Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
import sklearn.metrics as metrics
import math
class SimpleLinearRegression:
def get_predictions(self, X):
return np.dot(X, self._W)
 def _get_gradient_descent_step(self, X, targets, learning_rate):
predictions = self.get_predictions(X)
 error = predictions - targets
gradient = np.dot(X.T, error)/len(x)
 self._W -= learning_rate * gradient
def fit(self, X, y, iterations_num=1000, learning_rate=0.01):
self._W = np.zeros(X.shape[1])
 self._history_of_cost = []
self._w_history = [self._W]
for i in range(iterations_num):
predictions = self.get_predictions(X)
 cost = loss_func(predictions, y)
 self._history_of_cost.append(cost)
 self._get_gradient_descent_step(x, y, learning_rate)
 self._w_history.append(self._W.copy())
 return self 
<train_on_grid>
warnings.filterwarnings('ignore' )
house_price_linear_result = SimpleLinearRegression()
house_price_linear_result.fit(x, y, iterations_num=2000, learning_rate=0.01)
 print(house_price_linear_result._W )<train_on_grid>
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 c_test = test.copy()
c_train = train.copy()
x = df_train[['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF',
'YearBuilt', 'FullBath', 'OpenPorchSF', 'WoodDeckSF', '2ndFlrSF', 'YearRemodAdd',
'MasVnrArea', 'LotFrontage', 'LotArea']]
x =(x - x.mean())/ x.std()
x = np.c_[np.ones(x.shape[0]), x]
house_price_multi_variate_linear_result = SimpleLinearRegression()
house_price_multi_variate_linear_result.fit(x, y, iterations_num = 2000, learning_rate=0.01)
 print(house_price_multi_variate_linear_result._W )<define_variables>
c_train['train'] = 1
c_test['train'] = 0
df = pd.concat([c_train, c_test], axis=0,sort=False)
train_target_label = df_train['SalePrice']
top_feature_columns_modified = ['OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'YearBuilt', 'FullBath', 'OpenPorchSF',
'WoodDeckSF', '2ndFlrSF', 'YearRemodAdd', 'MasVnrArea', 'LotFrontage', 'LotArea']
training_sample_df = df_train[top_feature_columns_modified]
test_sample_df = df_test[top_feature_columns_modified]
training_sample_df.head()<categorify>
Null_val = [(c, df[c].isna().mean() *100)for c in df]
Null_val = pd.DataFrame(Null_val, columns=["column_name", "percentage"] )
imputer = SimpleImputer(strategy = 'median')
 imputer.fit(training_sample_df)
 imputer.fit(test_sample_df)
 training_sample_df = imputer.transform(training_sample_df)
 test_sample_df = imputer.transform(test_sample_df )<normalization>
Null_val = Null_val[Null_val.percentage > 50]
Null_val.sort_values("percentage", ascending=False)
scaler = StandardScaler()
scaler.fit(training_sample_df)
 scaler.fit(test_sample_df)
 training_sample_df = scaler.transform(training_sample_df)
 test_sample_df = scaler.transform(test_sample_df )<split>
df = df.drop(['Alley','PoolQC','Fence','MiscFeature'],axis=1)
X_train_split, X_test_split, y_train, y_test = train_test_split(training_sample_df, train_target_label, random_state=42, train_size = 0.8)
 X_train_split.shape<train_model>
object_columns_df = df.select_dtypes(include=['object'])
 numerical_columns_df =df.select_dtypes(exclude=['object'])
model = LinearRegression()
model.fit(training_sample_df, train_target_label )<compute_test_metric>
null_counts = object_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))
print('Intercept is ', model.intercept_)
 print("Slope i.e.coefficient of x is ", model.coef_ )<compute_test_metric>
columns_None = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','FireplaceQu','GarageCond']
object_columns_df[columns_None]= object_columns_df[columns_None].fillna('None')
print("Regression score is", model.score(training_sample_df, train_target_label))
print('train_target_label is ', train_target_label )<predict_on_test>
columns_with_lowNA = ['MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','Electrical','KitchenQual','Functional','SaleType']
object_columns_df[columns_with_lowNA] = object_columns_df[columns_with_lowNA].fillna(object_columns_df.mode().iloc[0])
y_predict = model.predict(X_test_split )<compute_test_metric>
null_counts = numerical_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))
rmse_simple_linear = np.sqrt(metrics.mean_squared_error(y_test, y_predict))
print('Root Mean Square Error is ', rmse_simple_linear)
 rmse_log_simple_linear = np.sqrt(mean_squared_log_error(y_test, y_predict))
print('Root Mean Square Log Error is ', rmse_log_simple_linear)
<prepare_output>
numerical_columns_df['GarageYrBlt'] = numerical_columns_df['GarageYrBlt'].fillna(numerical_columns_df['YrSold']-35)
 numerical_columns_df['LotFrontage'] = numerical_columns_df['LotFrontage'].fillna(68)
prediction_on_test_data = model.predict(test_sample_df)
 testID = df_test['Id']
predict_submission = pd.DataFrame()
predict_submission['ID'] = testID
predict_submission['SalePrice'] = prediction_on_test_data
predict_submission<compute_train_metric>
object_columns_df = object_columns_df.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1)
model_ridge = Ridge(alpha=0.5)
 model_ridge.fit(training_sample_df, train_target_label)
 y_predict_ridge = model_ridge.predict(X_test_split)
 mse_linear_ridge = np.sqrt(metrics.mean_squared_error(y_test, y_predict_ridge))
print('MSE of Linear Ridge is ', mse_linear_ridge)
 mse_log_linear_ridge = np.sqrt(mean_squared_log_error(y_test, y_predict_ridge))
print('MSE Log of Linear Ridge is ', mse_log_linear_ridge)
<compute_test_metric>
Neg_val = numerical_columns_df[numerical_columns_df['Age_House'] < 0]
Neg_val
model_lasso = Lasso(alpha=33)
 model_lasso.fit(training_sample_df, train_target_label)
 y_predict_lasso = model_lasso.predict(X_test_split)
 mse_linear_lasso = np.sqrt(metrics.mean_squared_error(y_test, y_predict_lasso))
print('MSE of Lasso Regression is ', mse_linear_lasso)
 mse_log_linear_lasso = np.sqrt(mean_squared_log_error(y_test, y_predict_lasso))
print('MSE Log of Lasso Regression is ', mse_log_linear_lasso )<compute_train_metric>
numerical_columns_df['TotalBsmtBath'] = numerical_columns_df['BsmtFullBath'] + numerical_columns_df['BsmtFullBath']*0.5
numerical_columns_df['TotalBath'] = numerical_columns_df['FullBath'] + numerical_columns_df['HalfBath']*0.5
numerical_columns_df['TotalSA']=numerical_columns_df['TotalBsmtSF'] + numerical_columns_df['1stFlrSF'] + numerical_columns_df['2ndFlrSF']
RFR = RandomForestRegressor(max_depth=50)
 RFR.fit(training_sample_df, train_target_label)
 y_predict_random_forest = RFR.predict(X_test_split)
 mse_random_forest = np.sqrt(metrics.mean_squared_error(y_test, y_predict_random_forest))
print('MSE Random Forest is ', mse_random_forest)
 mse_log_random_forest = np.sqrt(mean_squared_log_error(y_test, y_predict_random_forest))
print('MSE Log Random Forest is ', mse_log_random_forest )<create_dataframe>
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,"None" : 0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6
}
object_columns_df['ExterQual'] = object_columns_df['ExterQual'].map(bin_map)
 object_columns_df['ExterCond'] = object_columns_df['ExterCond'].map(bin_map)
 object_columns_df['BsmtCond'] = object_columns_df['BsmtCond'].map(bin_map)
 object_columns_df['BsmtQual'] = object_columns_df['BsmtQual'].map(bin_map)
 object_columns_df['HeatingQC'] = object_columns_df['HeatingQC'].map(bin_map)
 object_columns_df['KitchenQual'] = object_columns_df['KitchenQual'].map(bin_map)
 object_columns_df['FireplaceQu'] = object_columns_df['FireplaceQu'].map(bin_map)
 object_columns_df['GarageQual'] = object_columns_df['GarageQual'].map(bin_map)
 object_columns_df['GarageCond'] = object_columns_df['GarageCond'].map(bin_map)
 object_columns_df['CentralAir'] = object_columns_df['CentralAir'].map(bin_map)
 object_columns_df['LotShape'] = object_columns_df['LotShape'].map(bin_map)
 object_columns_df['BsmtExposure'] = object_columns_df['BsmtExposure'].map(bin_map)
 object_columns_df['BsmtFinType1'] = object_columns_df['BsmtFinType1'].map(bin_map)
 object_columns_df['BsmtFinType2'] = object_columns_df['BsmtFinType2'].map(bin_map)
 PavedDrive = {"N" : 0, "P" : 1, "Y" : 2}
object_columns_df['PavedDrive'] = object_columns_df['PavedDrive'].map(PavedDrive )
Y_test_predicted_for_submission = RFR.predict(test_sample_df)
 indexes = np.arange(df_test.shape[0]+2, 2*df_test.shape[0]+2)
 print('Indexex ', indexes)
<prepare_x_and_y>
rest_object_columns = object_columns_df.select_dtypes(include=['object'])
 object_columns_df = pd.get_dummies(object_columns_df, columns=rest_object_columns.columns)
all_features_except_target = [i for i in df_train.columns if i not in ['SalePrice'] ]
features_X = df_train[all_features_except_target]
y = df_train['SalePrice']
X_feature_engineering = features_X.copy()
X_feature_engineering['years_since_update'] = X_feature_engineering['YearRemodAdd'] - X_feature_engineering['YearBuilt']
X_feature_engineering['garage_value'] = X_feature_engineering['YearBuilt'] * X_feature_engineering['GarageCars']
X_feature_engineering = X_feature_engineering.drop(columns=['GarageCars'])
 feature_numerical_columns = [col_name for col_name in X_feature_engineering.columns if
X_feature_engineering[col_name].dtype in ['int64', 'float64']]<categorify>
df_final = pd.concat([object_columns_df, numerical_columns_df], axis=1,sort=False)
 df_final.head()
feature_categorical_cols = [col_name for col_name in X_feature_engineering.columns if
X_feature_engineering[col_name].nunique() < 50 and
X_feature_engineering[col_name].dtype in ['object', 'bool']]
feature_numerical_transformer = SimpleImputer(strategy='constant')
 feature_categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 feature_preprocessor = ColumnTransformer(
 transformers=[ 
('num', feature_numerical_transformer, feature_numerical_columns), 
('cat', feature_categorical_transformer, feature_categorical_cols)
 ] )<choose_model_class>
df_final = df_final.drop(['Id',],axis=1)
 df_train = df_final[df_final['train'] == 1]
df_train = df_train.drop(['train',],axis=1)
 df_test = df_final[df_final['train'] == 0]
df_test = df_test.drop(['SalePrice'],axis=1)
 df_test = df_test.drop(['train',],axis=1 )
feature_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0.0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.02, max_delta_step=0, max_depth=4,
min_child_weight=0.0, monotone_constraints='() ',
n_estimators=1250, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.8,
tree_method='exact', validate_parameters=1, verbosity=None)
 feature_classifier = Pipeline(steps=[('feature_preprocessor', feature_preprocessor), 
('feature_model', feature_model)
 ])
 feature_X_train, feature_X_valid, feature_y_train, feature_y_test = train_test_split(X_feature_engineering, y, random_state=0)
 feature_classifier.fit(feature_X_train, feature_y_train, feature_model__verbose=False)
 feature_predictions = feature_classifier.predict(feature_X_valid)
 print('RMSLE from custom-defined function', root_mean_squared_log_error(feature_y_test, feature_predictions))
print('RMSLE using scikit-learn function', mean_squared_log_error(feature_y_test, feature_predictions)) 
<save_to_csv>
target= df_train['SalePrice']
df_train = df_train.drop(['SalePrice'],axis=1)
X_test_original = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 X_test_original['years_since_update'] = X_test_original['YearRemodAdd'] - X_feature_engineering['YearBuilt']
X_test_original['garage_value'] = X_test_original['YearBuilt'] * X_test_original['GarageCars']
X_test_original = X_test_original.drop(columns=['GarageCars'])
 feature_classifier.fit(X_feature_engineering, y, feature_model__verbose=False)
 model_predictions = feature_classifier.predict(X_test_original)
 output = pd.DataFrame({'Id': X_test_original.Id,
'SalePrice': model_predictions})
 output.to_csv('submission.csv', index=False )<set_options>
x_train,x_test,y_train,y_test = train_test_split(df_train,target,test_size=0.33,random_state=0 )
pd.set_option('display.max_rows', None)
 pd.set_option('display.width', None)
 pd.set_option('display.max_colwidth', None)
 %matplotlib inline<load_from_csv>
xgb =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=12000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4,
)
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test1 = test.copy()
train1 = train.copy()<filter>
xgb.fit(x_train, y_train)
 lgbm.fit(x_train, y_train,eval_metric='rmse')
train1= train1[pd.notnull(train1['SalePrice'])]
train1.tail()<concatenate>
predict1 = xgb.predict(x_test)
 predict = lgbm.predict(x_test)
train1['train'] = 1
test1['train'] = 0
house = pd.concat([train1,test1], axis=0,sort=False)
 house.tail()<create_dataframe>
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict1))))
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict))))
NAN = [(c, house[c].isna().mean() *100)for c in house]
NAN = pd.DataFrame(NAN, columns=["column_name", "percentage"])
 NAN = NAN[NAN.percentage > 50]
NAN.sort_values("percentage", ascending=False )<drop_column>
xgb.fit(df_train, target)
 lgbm.fit(df_train, target,eval_metric='rmse')
house = house.drop(['Alley','PoolQC','Fence','MiscFeature'],axis=1 )<define_variables>
predict4 = lgbm.predict(df_test)
 predict3 = xgb.predict(df_test)
 predict_y =(predict3*0.45 + predict4 * 0.55)
object_columns_df = house.select_dtypes(include=['object'])
 numerical_columns_df =house.select_dtypes(exclude=['object'] )<count_missing_values>
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": predict_y
})
 submission.to_csv('submission.csv', index=False )