Kaggle competition: House Prices - Advanced Regression Techniques
results = np.zeros(( X_test.shape[0], 10))
for j in range(num_nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results, axis = 1)
 results = pd.Series(results, name = "Label")
 submission = pd.concat([pd.Series(range(1, 28001), name = "ImageId"), results], axis = 1)
 submission.ImageId = submission.ImageId.astype(int)
 submission.to_csv("prediction.csv", index = False )<import_modules>
train_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 test_data['SalePrice'] = np.nan
all_data = pd.concat([train_data, test_data])
 print("Rows in train data = {}".format(len(train_data)))
 print("Rows in test data = {}".format(len(test_data)) )
warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
print("NaNs in each training Feature")
 dfNull = all_data.isnull().sum().to_frame('nulls')
 print(dfNull.loc[dfNull['nulls'] > 0] )
data_train = pd.read_csv('.. /input/train.csv')
 print(data_train.columns)
 data_test = pd.read_csv('.. /input/test.csv' )<data_type_conversions>
all_data.drop(columns=['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature'], inplace=True )
elt_name = ['Alley','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu'
,'GarageType','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','MiscFeature']
data_train.update(data_train[elt_name].fillna('None'))
data_test.update(data_test[elt_name].fillna('None'))<sort_values>
mapping = {'Ex': 5.0, 'Gd': 4.0, 'TA': 3.0, 'Fa': 2.0, 'Po': 1.0}
all_data.replace({'KitchenQual': mapping}, inplace=True)
 all_data.replace({'BsmtQual': mapping}, inplace=True)
 all_data.replace({'BsmtCond': mapping}, inplace=True)
 all_data.replace({'GarageQual': mapping}, inplace=True)
 all_data.replace({'GarageCond': mapping}, inplace=True )
total = data_train.isnull().sum().sort_values(ascending=False)
 percent =(data_train.isnull().sum() /data_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total,percent],axis=1,keys=['Total','Percent'])
 missing_data.head(20)
<concatenate>
all_data['MSZoning'].fillna('RL', inplace=True)
 all_data.drop(columns=['Utilities'], inplace=True)
 all_data['Electrical'].fillna('SBrkr', inplace=True)
 all_data['Functional'].fillna('Typ', inplace=True )
total_test = data_test.isnull().sum().sort_values(ascending=False)
 percent_test =(data_test.isnull().sum() /data_test.isnull().count() ).sort_values(ascending=False)
 missing_data_test = pd.concat([total_test,percent_test],axis=1,keys=['Total','Percent'])
 missing_data_test.head(20 )<count_missing_values>
print(all_data['Exterior2nd'].values[all_data['Exterior1st'].isnull() == True] )
print(( missing_data[missing_data['Total'] > 1] ).index)
 data_train.update(data_train['Electrical'].fillna(data_train['Electrical'].dropna().mode().values.item()))
col_num=[f for f in data_train.columns if data_train.dtypes[f] != 'object']
data_train.update(data_train[col_num].fillna(0))
col_obj=[f for f in data_train.columns if data_train.dtypes[f] == 'object']
data_train.update(data_train[col_obj].fillna('None'))
data_train.isnull().sum().max()<count_missing_values>
all_data['Exterior1st'].fillna('VinylSd', inplace=True)
 all_data['Exterior2nd'].fillna('VinylSd', inplace=True )
col_num=[f for f in data_test.columns if data_test.dtypes[f] != 'object']
data_test.update(data_test[col_num].fillna(0))
col_obj=[f for f in data_test.columns if data_test.dtypes[f] == 'object']
data_test.update(data_test[col_obj].fillna('None'))
data_test.isnull().sum().max()<normalization>
print(all_data['MasVnrArea'].values[all_data['MasVnrType'].isnull() == True] )
saleprice_scaled = StandardScaler().fit_transform(data_train['SalePrice'][:,np.newaxis]);
low_range = saleprice_scaled[saleprice_scaled[:,0].argsort() ][:10]
high_range= saleprice_scaled[saleprice_scaled[:,0].argsort() ][-10:]
print('outer range(low)of the distribution:')
 print(low_range)
 print('
outer range(high)of the distribution:')
 print(high_range )<sort_values>
indices =(np.argwhere(( all_data['MasVnrType'].isnull().values)&(all_data['MasVnrArea'].isnull().values == False)) ).flatten()
print(indices)
 all_data.iloc[indices[0], all_data.columns.get_loc('MasVnrType')] = 'BrkFace'
all_data['MasVnrType'].fillna('None', inplace=True)
 all_data['MasVnrArea'].fillna(0.0, inplace=True )
outlier_ids = data_train.sort_values(by = 'GrLivArea', ascending = False)[:2]['Id']
print(outlier_ids.index)
 data_train = data_train.drop(outlier_ids.index)
 print(data_train.sort_values(by = 'GrLivArea', ascending = False)[:2][['SalePrice','GrLivArea']] )<feature_engineering>
all_data['BsmtCond'].fillna(3.0, inplace=True)
 all_data['BsmtQual'].fillna(3.0, inplace=True)
 all_data['BsmtExposure'].fillna('No', inplace=True)
 all_data['BsmtFinType1'].fillna('Unf', inplace=True)
 all_data['BsmtFinType2'].fillna('Unf', inplace=True)
 all_data['BsmtFullBath'].fillna(0.0, inplace=True)
 all_data['BsmtHalfBath'].fillna(0.0, inplace=True )
data_train['SalePrice'] = np.log(data_train['SalePrice'] )<feature_engineering>
meanKitchenQual = np.rint(all_data.groupby(['OverallQual'])['KitchenQual'].mean())
 print(meanKitchenQual)
 print(all_data['OverallQual'].values[np.isnan(all_data['KitchenQual'].values)])
 all_data['KitchenQual'].fillna(3.0, inplace=True )
data_train['GrLivArea'] = np.log(data_train['GrLivArea'])
 data_test['GrLivArea'] = np.log(data_test['GrLivArea'] )<feature_engineering>
all_data['GarageType'].fillna('Attchd', inplace=True)
 all_data['GarageFinish'].fillna('Unf', inplace=True)
 all_data['GarageCars'].fillna(2.0, inplace=True)
 all_data['GarageQual'].fillna(3.0, inplace=True)
 all_data['GarageCond'].fillna(3.0, inplace=True )
data_train['HasBsmt'] = pd.Series(len(data_train['TotalBsmtSF']), index=data_train.index)
 data_train['HasBsmt'] = 0
data_train.loc[data_train['TotalBsmtSF']>0,'HasBsmt'] = 1
data_test['HasBsmt'] = pd.Series(len(data_test['TotalBsmtSF']), index=data_test.index)
 data_test['HasBsmt'] = 0
data_test.loc[data_test['TotalBsmtSF']>0,'HasBsmt'] = 1<feature_engineering>
for i in keys:
print(np.argwhere(np.isnan(all_data[i].values)) )
data_train.loc[data_train['HasBsmt']==1,'TotalBsmtSF'] = np.log(data_train['TotalBsmtSF'])
 data_test.loc[data_test['HasBsmt']==1,'TotalBsmtSF'] = np.log(data_test['TotalBsmtSF'] )<categorify>
median_total = np.median(all_data['TotalBsmtSF'].values[~np.isnan(all_data['TotalBsmtSF'].values)])
 median_SF1 = np.median(all_data['BsmtFinSF1'].values[~np.isnan(all_data['BsmtFinSF1'].values)])
 median_Unf = np.median(all_data['BsmtUnfSF'].values[~np.isnan(all_data['BsmtUnfSF'].values)])
 print(median_total, median_SF1, median_Unf)
 all_data['BsmtFinSF1'].fillna(median_SF1, inplace=True)
 all_data['BsmtFinSF2'].fillna(0.0, inplace=True)
 all_data['BsmtUnfSF'].fillna(median_Unf, inplace=True)
 all_data['TotalBsmtSF'].fillna(median_total, inplace=True )
data_all = pd.concat([data_train.drop(['SalePrice'], axis=1), data_test] ).reset_index(drop=True)
 data_all = pd.get_dummies(data_all)
 data_all.drop(['Id'],axis=1,inplace=True)
<prepare_x_and_y>
garage_yr_built = all_data['GarageYrBlt'].values
indices = np.argwhere(np.isnan(garage_yr_built))
yearbuilt = all_data['YearBuilt'].values[indices]
garage_yr_built[indices] = yearbuilt
all_data['GarageYrBlt'] = garage_yr_built
median = np.median(all_data['GarageArea'].values[~np.isnan(all_data['GarageArea'].values)])
 print(median)
 all_data['GarageArea'].fillna(median, inplace=True )
y = data_train['SalePrice'].reset_index(drop=True)
 x_train = data_all.iloc[:len(y), :]
x_test = data_all.iloc[len(y):, :]
train = data_all[:len(y)]
test = data_all[len(y):]
x_train.shape, x_test.shape, y.shape,train.shape,test.shape<compute_train_metric>
median = np.median(all_data['LotFrontage'].values[~np.isnan(all_data['LotFrontage'].values)])
 print(median)
 all_data['LotFrontage'].fillna(median, inplace=True )
print('START ML', datetime.now() ,)
 kfolds = KFold(n_splits=5, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, x=x_train, y=y):
rmse = np.sqrt(-cross_val_score(model, x, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
print(all_data.isnull().sum().values )
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]<choose_model_class>
mapping = {1: 'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul',
8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'}
all_data.replace({'MoSold': mapping}, inplace=True)
 mapping = {20:'A', 30:'B', 40:'C', 45:'D', 50:'E', 60:'F', 70:'G', 75:'H', 80:'I', 85:'J',
90:'K', 120:'L', 150:'M', 160:'N', 180:'O', 190:'P'}
all_data.replace({'MSSubClass': mapping}, inplace=True)
 all_data['sale_age'] = 2020 - all_data['YrSold']
all_data['house_age'] = 2020 - all_data['YearBuilt']
all_data['remodel_age'] = 2020 - all_data['YearRemodAdd']
all_data['garage_age'] = 2020 - all_data['GarageYrBlt']
all_data.drop(columns=['YrSold', 'YearBuilt', 'YearRemodAdd', 'GarageYrBlt'], inplace=True)
 all_data['TotalArea'] =(all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
+ all_data['GrLivArea'] + all_data['GarageArea'])
 all_data['TotalBathrooms'] = all_data['FullBath'] + all_data['HalfBath']*0.5
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio)) 
<choose_model_class>
columns = ['MSZoning', 'MSSubClass', 'PavedDrive', 'GarageFinish', 'Foundation', 'Functional', 'LandContour', 'Condition1',
'Condition2', 'Street', 'LotShape', 'ExterQual', 'ExterCond', 'LotConfig', 'LandSlope', 'Neighborhood','BldgType',
'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st','Exterior2nd', 'MasVnrType', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'GarageType', 'SaleType', 'SaleCondition',
'MoSold']
one_hot = pd.get_dummies(all_data.loc[:, columns], drop_first=True)
 all_data.drop(columns=columns, inplace=True)
 for i in range(len(one_hot.columns.values)) :
all_data[one_hot.columns.values[i]] = one_hot[one_hot.columns.values[i]].values
gbr = GradientBoostingRegressor(n_estimators=2000, learning_rate=0.01, max_depth=30, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )<choose_model_class>
predict_data = all_data.loc[np.isnan(all_data['SalePrice'].values)]
model_data = all_data.loc[~np.isnan(all_data['SalePrice'].values)]
predict_ids = predict_data['Id'].values
predict_data.drop(columns=['Id'], inplace=True)
 model_data.drop(columns=['Id'], inplace=True )
random_forest = RandomForestRegressor(n_estimators=2000,
max_depth=30,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
random_state=10,
oob_score=True
)<choose_model_class>
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(model_data,1,["SalePrice", 'LotArea', 'GarageArea'])
 model_data = model_data.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )<choose_model_class>
scaler = MinMaxScaler()
scaler.fit(model_data.drop(columns=['SalePrice']))
X_train = scaler.transform(model_data.drop(columns=['SalePrice']))
y_train = model_data['SalePrice']
X_predict_data_scaled = scaler.transform(predict_data.drop(columns=['SalePrice']))
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, random_forest),
meta_regressor=gbr,
use_features_in_secondary=True )<compute_test_metric>
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
print('TEST score on CV')
 names = ["Kernel Ridge", "Lasso", "ElasticNet", "xgboost", "gbr", "stack_gen"]
models= [ridge, lasso, elasticnet, xgboost, gbr, stack_gen]
for name, model in zip(names, models):
score = cv_rmse(model)
 print("{} score: {:.6f}({:.4f})".format(name,score.mean() ,score.std()))<predict_on_test>
param_grid = {
'criterion' : ['mse'],
'n_estimators': [90, 100, 110],
'max_features': ['auto', 'log2'],
'max_depth' : [7, 9, 11, 13]
}
randomForest_CV = GridSearchCV(estimator = RandomForestRegressor() , param_grid = param_grid, cv = 3)
 randomForest_CV.fit(X_train, y_train)
 print(randomForest_CV.best_params_)
 print(randomForest_CV.best_score_)
 criterion = randomForest_CV.best_params_.get('criterion')
 nRF = randomForest_CV.best_params_.get('n_estimators')
 max_features = randomForest_CV.best_params_.get('max_features')
 max_depth = randomForest_CV.best_params_.get('max_depth')
 bestRF = RandomForestRegressor(n_estimators=nRF, max_depth=max_depth, max_features=max_features, criterion=criterion)
 bestRF.fit(X_train, y_train )
class MixModelWeight() :
def __init__(self,mod,weight):
self.mod = mod
self.weight = weight
def fit(self,X,y):
for model in self.mod:
model.fit(X,y)
 return self
def predict(self,X):
w = list()
pred = np.array([model.predict(X)for model in self.mod])
 for data in range(pred.shape[1]):
single = [pred[model,data]*weight for model,weight in zip(range(pred.shape[0]),self.weight)]
w.append(np.sum(single))
return w
mix_model = MixModelWeight(mod = [elasticnet,lasso,ridge,xgboost,stack_gen],weight=[0.1,0.1,0.1,0.1,0.6])
 print('START fit model')
 mix_model.fit(x_train,y)
 print('RMSLE score on train data:')
 score = rmsle(y, mix_model.predict(x_train))
print(score )<prepare_output>
xgboost = xgb.XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3)
 xgboost.fit(X_train,y_train )
print('Predict submission', datetime.now() ,)
 submission=pd.DataFrame({'Id':data_test['Id'], 'SalePrice':np.floor(np.exp(mix_model.predict(x_test)))})
 submission_stack=pd.DataFrame({'Id':data_test['Id'], 'SalePrice':np.floor(np.exp(stack_gen.predict(x_test)))})
 submission_mix=pd.DataFrame({'Id':data_test['Id'], 'SalePrice':(submission['SalePrice'])*0.8+(submission_stack['SalePrice']*0.2)})
 print(submission )<save_to_csv>
testPredictionsRF = bestRF.predict(X_predict_data_scaled)
 testPredictionsXGB = xgboost.predict(X_predict_data_scaled)
 outputRF = pd.DataFrame({'Id': predict_ids, 'SalePrice': testPredictionsRF})
 outputRF.to_csv('my_submissionRF.csv', index=False)
 outputXGB = pd.DataFrame({'Id': predict_ids, 'SalePrice': testPredictionsXGB})
 outputXGB.to_csv('my_submissionXGB.csv', index=False )