Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<create_dataframe>
warnings.filterwarnings('ignore')
 get_ipython().magic(u'matplotlib inline')
 plt.style.use('ggplot')
 print("Modules imported
")
 print("Files in current directory:")
 print(check_output(["ls", ".. /input"] ).decode("utf8"))
test_na =(test.isnull().sum() / len(test)) * 100
test_na = test_na.drop(test_na[test_na==0].index ).sort_values(ascending=False)
 test_na_df = pd.DataFrame({'Missing Ratio' :test_na})
 test_na_df<data_type_conversions>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print("train ", train.shape)
 print("test ", test.shape )
train['PoolQC'] = train['PoolQC'].fillna('None')
 test['PoolQC'] = test['PoolQC'].fillna('None' )<data_type_conversions>
print(train["MSSubClass"].value_counts())
train["MiscFeature"] = train["MiscFeature"].fillna("None")
 test["MiscFeature"] = test["MiscFeature"].fillna("None" )<data_type_conversions>
trainWprice = pd.DataFrame(train)
 trainNoPrice = trainWprice.drop("SalePrice", axis=1)
 full=pd.concat([trainNoPrice,test], ignore_index=True)
 full.drop(['Id'],axis=1, inplace=True)
 print("train ", train.shape)
 print("test ", test.shape)
 print("full without Id and no SalePrice ", full.shape )
train["Alley"] = train["Alley"].fillna("None")
 test["Alley"] = test["Alley"].fillna("None" )<data_type_conversions>
class feat_eng(BaseEstimator, TransformerMixin):
def __init__(self, fill_missvals = True):
self.fill_missvals = fill_missvals
pass
def fit(self,X,y=None):
return self
def transform(self,X):
if self.fill_missvals:
X["PoolQC"] = X["PoolQC"].fillna("None")
 X["MiscFeature"] = X["MiscFeature"].fillna("None")
 X["Alley"] = X["Alley"].fillna("None")
 X["Fence"] = X["Fence"].fillna("None")
 X["FireplaceQu"] = X["FireplaceQu"].fillna("None")
 X["LotFrontage"] = X.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
X['MSZoning'] = X['MSZoning'].fillna(X['MSZoning'].mode() [0])
 X["Functional"] = X["Functional"].fillna("Typ")
 X['Electrical'] = X['Electrical'].fillna(X['Electrical'].mode() [0])
 X['KitchenQual'] = X['KitchenQual'].fillna(X['KitchenQual'].mode() [0])
 X['Exterior1st'] = X['Exterior1st'].fillna(X['Exterior1st'].mode() [0])
 X['Exterior2nd'] = X['Exterior2nd'].fillna(X['Exterior2nd'].mode() [0])
 X['SaleType'] = X['SaleType'].fillna(X['SaleType'].mode() [0])
 X['MSSubClass'] = X['MSSubClass'].fillna("None")
 for col in('GarageType', 'GarageFinish', 'GarageQual',
'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2','MasVnrType'):
X[col] = X[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars', 'MasVnrArea'):
X[col] = X[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
X[col] = X[col].fillna(0)
 X['MSSubClass'] = X['MSSubClass'].apply(str)
 X['OverallCond'] = X['OverallCond'].astype(str)
 X['YrSold'] = X['YrSold'].astype(str)
 X['MoSold'] = X['MoSold'].astype(str)
 X = X.drop(['Utilities'], axis=1)
 return X
train["Fence"] = train["Fence"].fillna("None")
 test["Fence"] = test["Fence"].fillna("None" )<data_type_conversions>
class add_feature(BaseEstimator, TransformerMixin):
def __init__(self,additional=1):
self.additional = additional
def fit(self,X,y=None):
return self
def transform(self,X):
if self.additional==1:
X["TotalHouse"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"]
X["TotalArea"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"] + X["GarageArea"]
else:
X["TotalHouse"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"]
X["TotalArea"] = X["TotalBsmtSF"] + X["1stFlrSF"] + X["2ndFlrSF"] + X["GarageArea"]
return X
train["FireplaceQu"] = train["FireplaceQu"].fillna("None")
 test["FireplaceQu"] = test["FireplaceQu"].fillna("None" )<categorify>
class skew_dummies(BaseEstimator, TransformerMixin):
def __init__(self,skew=0.75):
self.skew = skew
def fit(self,X,y=None):
return self
def transform(self,X):
X_numeric=X.select_dtypes(exclude=["object"])
 skewness = X_numeric.apply(lambda x: skew(x))
skewness_features = skewness[abs(skewness)>= self.skew].index
X[skewness_features] = np.log1p(X[skewness_features])
 X = pd.get_dummies(X)
 return X
mapper = train.groupby("Neighborhood" ).median() ['LotFrontage'].to_dict()
for k,v in mapper.items() :
train.loc[(train['LotFrontage'].isnull() == True)&(train['Neighborhood'] == k), 'LotFrontage'] = v
test.loc[(test['LotFrontage'].isnull() == True)&(test['Neighborhood'] == k), 'LotFrontage'] = v<data_type_conversions>
class labelenc(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self,X,y=None):
return self
def transform(self,X):
lab = LabelEncoder()
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
X[c] = lab.fit_transform(X[c])
 return X
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None' )<data_type_conversions>
class grid() :
def __init__(self,model):
self.model = model
def grid_get(self,X,y,param_grid):
grid_search = GridSearchCV(self.model,param_grid,cv=5, scoring="neg_mean_squared_error")
 grid_search.fit(X,y)
 print(grid_search.best_params_, np.sqrt(-grid_search.best_score_))
grid_search.cv_results_['mean_test_score'] = np.sqrt(-grid_search.cv_results_['mean_test_score'])
 print(pd.DataFrame(grid_search.cv_results_)[['params','mean_test_score','std_test_score']] )
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
train[col] = train[col].fillna(0)
 test[col] = test[col].fillna(0 )<data_type_conversions>
class drop_cols(BaseEstimator, TransformerMixin):
def __init__(self, remove_cols = True):
self.remove_cols = remove_cols
pass
def fit(self,X,y=None):
return self
def transform(self,X):
if self.remove_cols:
del X['PoolQC']
del X['BsmtFinType1']
del X['LowQualFinSF']
del X['MoSold']
del X['Electrical']
del X['BldgType']
del X['SaleType']
del X['BsmtFinSF2']
del X['Exterior2nd']
del X['ExterCond']
del X['2ndFlrSF']
del X['3SsnPorch']
del X['Exterior1st']
del X['MasVnrType']
del X['GarageFinish']
del X['Alley']
del X['PoolArea']
del X['RoofStyle']
del X['MiscFeature']
del X['Fence']
del X['BsmtHalfBath']
del X['Street']
del X['LotConfig']
del X['TotalBsmtSF']
del X['GarageYrBlt']
return X
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
train[col] = train[col].fillna(0)
 test[col] = test[col].fillna(0 )<data_type_conversions>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None' )<data_type_conversions>
pipe = Pipeline([ 
('feat_eng', feat_eng()), 
('add_feature', add_feature(additional=2)) , 
('lab_enc', labelenc()), 
('drop_cols', drop_cols()), 
('skew_dummies', skew_dummies(skew=1)) ,
])
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 trainWprice = pd.DataFrame(train)
 trainNoPrice = trainWprice.drop("SalePrice", axis=1)
 full=pd.concat([trainNoPrice,test], ignore_index=True)
 full.drop(['Id'],axis=1, inplace=True)
 print("full without Id and no price ", full.shape)
 FullDataPipe = pipe.fit_transform(full)
 print("FullDataPipe ", FullDataPipe.shape)
 n_train=train.shape[0]
trainFinal = pd.DataFrame(FullDataPipe[:n_train])
 testFinal = pd.DataFrame(FullDataPipe[n_train:])
 y= train.SalePrice
yFinal = np.log(train.SalePrice)
 scaler = RobustScaler()
trainFinal = scaler.fit_transform(trainFinal)
 testFinal = scaler.fit_transform(testFinal)
 print("trainFinal", trainFinal.shape)
 print("testFinal", testFinal.shape)
 print("yFinal", yFinal.shape )
train["MasVnrType"] = train["MasVnrType"].fillna("None")
 test["MasVnrType"] = test["MasVnrType"].fillna("None")
 train["MasVnrArea"] = train["MasVnrArea"].fillna(0)
 test["MasVnrArea"] = test["MasVnrArea"].fillna(0 )<categorify>
trainFinalFI = pd.DataFrame(trainFinal)
 yFinalFI = yFinal
lasso=Lasso(alpha=0.001, copy_X=True, fit_intercept=True, max_iter=1000,
normalize=False, positive=False, precompute=False, random_state=None,
selection='cyclic', tol=0.0001, warm_start=False)
 lasso.fit(trainFinalFI,yFinalFI)
 FI_lasso = pd.DataFrame({"Feature Importance":lasso.coef_}, index=trainFinalFI.columns)
 FI_lasso[FI_lasso["Feature Importance"]!=0].sort_values("Feature Importance" ).plot(kind="barh",figsize=(15,25))
plt.xticks(rotation=90)
 plt.show()
train['MSZoning'] = train['MSZoning'].fillna(train['MSZoning'].mode().item())
 test['MSZoning'] = test['MSZoning'].fillna(test['MSZoning'].mode().item() )<drop_column>
trainFinal = pd.DataFrame(trainFinal)
 ColsMissingValues = trainFinal.isnull().sum()
print("There are ", len(ColsMissingValues[ColsMissingValues>0]), " features with missing values")
 all_data_na =(trainFinal.isnull().sum() / len(trainFinal)) * 100
all_data_na = all_data_na.sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 print(missing_data.head(len(ColsMissingValues[ColsMissingValues>0])))
 print("_"*80)
 print("trainFinal ", trainFinal.shape)
 print("yFinal ", yFinal.shape )
train.drop('Utilities',axis=1,inplace=True)
 test.drop('Utilities',axis=1,inplace=True )<data_type_conversions>
def rmse_cv(model,X,y):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=5))
return rmse
train["Functional"] = train["Functional"].fillna("Typ")
 test["Functional"] = test["Functional"].fillna("Typ" )<categorify>
train['Electrical'] = train['Electrical'].fillna(train['Electrical'].mode() [0])
 test['Electrical'] = test['Electrical'].fillna(test['Electrical'].mode() [0] )<data_type_conversions>
models = [
Lasso(alpha =0.0005, random_state=1),
ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3),
KernelRidge(alpha=2.2, coef0=2.5, degree=3, gamma=None, kernel='polynomial',kernel_params=None),
SVR(C=2, cache_size=200, coef0=0.1, degree=3, epsilon=0.005, gamma=0.005,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.01, verbose=False),
GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,
learning_rate=0.05, loss='huber', max_depth=5,
max_features='sqrt', max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=10, min_samples_split=10,
min_weight_fraction_leaf=0.0, n_estimators=2000,
n_iter_no_change=None, presort='auto', random_state=None,
subsample=1.0, tol=0.0001, validation_fraction=0.1, verbose=0,
warm_start=False),
]
names = ["LASSO", "ELA","KER", "SVR ", "GBR" ]
train['KitchenQual'] = train['KitchenQual'].fillna(train['KitchenQual'].mode() [0])
 test['KitchenQual'] = test['KitchenQual'].fillna(test['KitchenQual'].mode() [0] )<categorify>
ModScores = {}
for name, model in zip(names, models):
score = rmse_cv(model, trainFinal, yFinal)
 ModScores[name] = score.mean()
print("{}: {:.6f}".format(name,score.mean()))
print("trainFinal ", trainFinal.shape)
 print("_"*80)
 for key, value in sorted(ModScores.items() , key = itemgetter(1), reverse = False):
print(key, value )
for col in ['Exterior1st','Exterior2nd']:
train[col] = train[col].fillna(train[col].mode() [0])
 test[col] = test[col].fillna(test[col].mode() [0] )<data_type_conversions>
param_grid = [{'kernel': ["rbf"], 'degree': [3]},
{'gamma': [0.005],'coef0': [0.1,0.05],'tol': [0.01],
'C': [2.5,2.2],'epsilon': [0.005],},]
train['SaleType'] = train['SaleType'].fillna(train['SaleType'].mode() [0])
 test['SaleType'] = test['SaleType'].fillna(test['SaleType'].mode() [0] )<data_type_conversions>
model4cv = SVR()
grid_search = GridSearchCV(model4cv, param_grid)
 grid_search.fit(trainFinal, yFinal)
 print(grid_search.best_params_)
 print(grid_search.best_estimator_)
 print("_"*80)
 print(grid(grid_search.best_estimator_ ).grid_get(trainFinal, yFinal,{}))
train['MSSubClass'] = train['MSSubClass'].fillna("None")
 test['MSSubClass'] = test['MSSubClass'].fillna("None" )<create_dataframe>
lasso = Lasso(alpha =0.0005, random_state=1)
 ker = KernelRidge(alpha=2.2, coef0=2.5, degree=3, gamma=None, kernel='polynomial',
kernel_params=None)
 svr = SVR(C=2, cache_size=200, coef0=0.1, degree=3, epsilon=0.005, gamma=0.005,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.01, verbose=False)
 gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 xgb = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 ela = ElasticNet(alpha=0.01, copy_X=True, fit_intercept=True, l1_ratio=0.05,
max_iter=5000, normalize=False, positive=False, precompute=False,
random_state=None, selection='cyclic', tol=0.0001, warm_start=False)
train_na =(train.isnull().sum() / len(train)) * 100
train_na = train_na.drop(train_na[train_na==0].index ).sort_values(ascending=False)
 train_na_df = pd.DataFrame({'Missing Ratio' :train_na})
 train_na_df<create_dataframe>
model = SVR(C=2, cache_size=200, coef0=0.1, degree=3, epsilon=0.005, gamma=0.005,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.01, verbose=False)
 model.fit(trainFinal, yFinal)
 score = rmse_cv(model, trainFinal, yFinal)
 print(" model score: {:.5f}({:.4f})
 ".format(score.mean() , score.std()))
pred = np.exp(model.predict(testFinal))
pred = np.around(pred, decimals=4, out=None)
 print(pred )
test_na =(test.isnull().sum() / len(test)) * 100
test_na = test_na.drop(test_na[test_na==0].index ).sort_values(ascending=False)
 test_na_df = pd.DataFrame({'Missing Ratio' :test_na})
 test_na_df<data_type_conversions>
result=pd.DataFrame({'Id':test.Id, 'SalePrice':pred})
 result.to_csv("submission.csv",index=False)
 print(result )