Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
%matplotlib inline
warnings.filterwarnings("ignore" )
random_forest = RandomForestClassifier(n_estimators = 100)
 random_forest.fit(X_train, y_train)
 print("Random Forest Classifier model")
 print("="*30)
 print("Train Accuracy:", random_forest.score(X_train, y_train))
print("Test Accuracy:", random_forest.score(X_test, y_test))<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train = train.drop('Id', axis = 1)
 print(train.shape)
 print(test.shape )
xgb = XGBClassifier()
xgb.fit(X_train, y_train)
 print("XGB Classifier model")
 print("="*20)
 print("Train Accuracy:", xgb.score(X_train, y_train))
print("Test Accuracy:", xgb.score(X_test, y_test))<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
validation_df<drop_column>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
X_validate = validation_df.drop("PassengerId", axis = 1 )<predict_on_test>
missing = train.isnull().sum().sort_values(ascending = False)
 missing_pct = missing /len(train)
 features_to_discard = list(missing_pct[missing_pct > 0.8].index)
 print('features discared:', features_to_discard)
 train.drop(features_to_discard, axis=1, inplace = True )
dt_model.fit(X, y)
 prediction = dt_model.predict(X_validate)
 prediction<save_to_csv>
nominal_features = ['MSSubClass', 'MSZoning', 'LandContour', 'LotConfig', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType',
'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'Heating',
'GarageType', 'SaleType', 'SaleCondition']
ordinal_features = ['Street', 'CentralAir', 'LotShape', 'Utilities', 'LandSlope', 'OverallQual', 'OverallCond', 'ExterCond', 'ExterQual',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'Electrical', 'KitchenQual',
'Functional', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive']
discrete_features = ['YearBuilt', 'YearRemodAdd', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr',
'TotRmsAbvGrd', 'Fireplaces', 'GarageYrBlt', 'GarageCars', 'YrSold', ]
continuous_features = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch',
'PoolArea', 'MiscVal', ]
misc_features = ['MoSold', ]
output = pd.DataFrame({'PassengerId': validation_df.PassengerId, 'Survived': prediction})
 output.to_csv('my_submission.csv', index = False )<set_options>
oridnal_map = {
'No':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5,
'N':0, 'Y': 2,
'MnWw':1, 'GdWo':2, 'MnPrv':3, 'GdPrv':4,
'IR3':1, 'IR2':2, 'IR1':3, 'Reg':4,
'ELO':1, 'NoSeWa':2, 'NoSewr':3, 'AllPub':4,
'Sev':1, 'Mod':2, 'Gtl':3,
'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6,
'Mix':1, 'FuseP':2, 'FuseF':3, 'FuseA':4, 'SBrkr':5,
'Sal':1, 'Sev':2, 'Maj2':3, 'Maj1':4, 'Mod':5, 'Min2':6, 'Min1':7, 'Typ':8,
'Unf':1, 'RFn':2, 'Fin':3,
'N':0, 'P':1, 'Y':2,
'Grvl':1, 'Pave':2,
'Mn':2, 'Av':3, 'Gd':4
}
class OrdinalTransformer(BaseEstimator, TransformerMixin):
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
df_x = pd.DataFrame(X, columns=ordinal_features)
 df_x = df_x.fillna('No')
 df_x = df_x.applymap(lambda x:oridnal_map.get(x,x))
return df_x
warnings.filterwarnings("ignore" )<load_from_csv>
ordinal_transformer = OrdinalTransformer()
x =ordinal_transformer.fit_transform(train[ordinal_features])
 x
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test["SalePrice"] = np.nan<define_variables>
train[discrete_features].isnull().sum()
num_discrete = ['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd',
'Fireplaces','GarageCars','GarageYrBlt','YearBuilt','YearRemodAdd','YrSold','MoSold']
num_continuous = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF',
'2ndFlrSF','LowQualFinSF','GrLivArea','GarageArea','WoodDeckSF','OpenPorchSF','EnclosedPorch',
'3SsnPorch','ScreenPorch','PoolArea','MiscVal','SalePrice']
cat_ordinal = ['LotShape','Utilities','LandSlope','OverallQual','OverallCond','ExterQual','ExterCond',
'BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir',
'Electrical','KitchenQual','Functional','FireplaceQu','GarageFinish','GarageQual','GarageCond',
'PavedDrive','PoolQC','Fence']
cat_nominal = ['MSSubClass','MSZoning','Street','Alley','LandContour','LotConfig','Neighborhood',
'Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st',
'Exterior2nd','MasVnrType','Foundation','Heating','GarageType','MiscFeature',
'SaleType','SaleCondition']<compute_test_metric>
discrete_imputer = Imputer(strategy='most_frequent')
 X = discrete_imputer.fit_transform(train[discrete_features])
 df_x = pd.DataFrame(X, columns=discrete_features)
 df_x.isnull().sum()
def z_score(df):
return(df-df.mean())/df.std(ddof=0 )<create_dataframe>
train[continuous_features].isnull().sum()
idx = []
outliers = []
corrs = []
zeros = []
for i in num_continuous:
if str(train[i].dtype)!= 'object':
idx.append(i)
 outliers.append(list(abs(z_score(train[i])) > 3 ).count(True))
corrs.append(train.SalePrice.corr(train[i]))
zeros.append(len(train[i][train[i] == 0])/len(train[i]))
outs = pd.DataFrame({'
'Feature': idx,
'Corr': corrs,
'% Zeros': zeros} ).sort_values(ascending = False,
by = '
outs = outs[outs["
outs = outs[abs(outs["Corr"])> 0.3].reset_index(drop=True)
 outs = outs[outs["% Zeros"] < 0.3].reset_index(drop=True)
 outs<drop_column>
continuous_inputer = Imputer(strategy='mean')
 X = continuous_inputer.fit_transform(train[continuous_features])
 df_x = pd.DataFrame(X, columns=continuous_features)
 print(df_x.isnull().sum())
 df_x
train = train.drop(train[(train['GrLivArea'] > 4000)&(train['SalePrice'] < 300000)].index )<drop_column>
continuous_inputer.statistics_
train = train.drop(train[(train['LotFrontage'] > 300)&(train['SalePrice'] < 300000)].index )<concatenate>
std_transformer = StandardScaler()
X = std_transformer.fit_transform(df_x)
 df_x = pd.DataFrame(X, columns=continuous_features)
 df_x
dataset = pd.concat([train,test],axis=0 ).reset_index(drop=True)
 dataset = dataset.fillna(np.nan )<count_missing_values>
class CatImputer(BaseEstimator, TransformerMixin):

 def fit(self, X, y=None):
self.fill = pd.Series([X[c].value_counts().index[0]
if X[c].dtype == np.dtype('O')else X[c].mean() for c in X],
index=X.columns)
 return self
def transform(self, X, y=None):
return X.fillna(self.fill)
 nominal_pipeline = Pipeline(steps=[ 
('imputer', CatImputer()), 
('encoder', OneHotEncoder(sparse=False, handle_unknown='ignore'))
])
def show_null(df):
null_columns =(df.isnull().sum(axis = 0)/len(df)).sort_values(ascending=False ).index
null_data = pd.concat([df.isnull().sum(axis = 0), 
(df.isnull().sum(axis = 0)/len(df)).sort_values(ascending=False),
df.loc[:, df.columns.isin(list(null_columns)) ].dtypes]
, axis=1)
 null_data = null_data.rename(columns={0: '
1: '% null',
2: 'type'} ).sort_values(ascending=False, by = '% null')
 null_data = null_data[null_data["
return null_data
show_null(dataset )<filter>
train[nominal_features].isnull().sum()
dataset.PoolQC[dataset.PoolQC.isnull() == True] = 'NA'
dataset.Fence[dataset.Fence.isnull() == True] = 'NA'
dataset.FireplaceQu[dataset.FireplaceQu.isnull() == True] = 'NA'
dataset.GarageCond[dataset.GarageCond.isnull() == True] = 'NA'
dataset.GarageQual[dataset.GarageQual.isnull() == True] = 'NA'
dataset.GarageFinish[dataset.GarageFinish.isnull() == True] = 'NA'
dataset.BsmtExposure[dataset.BsmtExposure.isnull() == True] = 'NA'
dataset.BsmtCond[dataset.BsmtCond.isnull() == True] = 'NA'
dataset.BsmtQual[dataset.BsmtQual.isnull() == True] = 'NA'
dataset.BsmtFinType2[dataset.BsmtFinType2.isnull() == True] = 'NA'
dataset.BsmtFinType1[dataset.BsmtFinType1.isnull() == True] = 'NA'
dataset.Electrical[dataset.Electrical.isnull() == True] = stats.mode(train.Electrical)[0][0]
dataset.Functional[dataset.Functional.isnull() == True] = stats.mode(train.Functional)[0][0]
dataset.KitchenQual[dataset.KitchenQual.isnull() == True] = stats.mode(train.KitchenQual)[0][0]
dataset.Utilities[dataset.Utilities.isnull() == True] = stats.mode(train.Utilities)[0][0]<feature_engineering>
nominal_X = nominal_pipeline.fit_transform(train[nominal_features] )
dataset.MiscFeature[dataset.MiscFeature.isnull() == True] = 'NA'
dataset.Alley[dataset.Alley.isnull() == True] = 'NA'
dataset.GarageType[dataset.GarageType.isnull() == True] = 'NA'
dataset.MasVnrType[dataset.MasVnrType.isnull() == True] = 'NA'
dataset.MSZoning[dataset.MSZoning.isnull() == True] = stats.mode(train.MSZoning)[0][0]
dataset.SaleType[dataset.SaleType.isnull() == True] = stats.mode(train.SaleType)[0][0]
dataset.Exterior1st[dataset.Exterior1st.isnull() == True] = stats.mode(train.Exterior1st)[0][0]
dataset.Exterior2nd[dataset.Exterior2nd.isnull() == True] = stats.mode(train.Exterior2nd)[0][0]<feature_engineering>
continous_pipeline = Pipeline(steps=[ 
('imputer', Imputer(strategy='mean')) , 
('log1p', FunctionTransformer(np.log1p)) , 
('std_scaler', StandardScaler()),
])
 ordinal_pipeline = Pipeline(steps=[ 
('transformer', OrdinalTransformer()),
])
 discrete_pipeline = Pipeline(steps=[ 
('imputer', Imputer(strategy='most_frequent')) ,
])
 nominal_pipeline = Pipeline(steps=[ 
('imputer', CatImputer()), 
('encoder', OneHotEncoder(sparse=False, handle_unknown='ignore'))
])
 prepare_pipeline = ColumnTransformer([ 
('ordinal', ordinal_pipeline, ordinal_features), 
('discrete', discrete_pipeline, discrete_features), 
('continuous', continous_pipeline, continuous_features), 
('nominal', nominal_pipeline, nominal_features),
] )
dataset.BsmtFullBath[dataset.BsmtFullBath.isnull() == True] = stats.mode(train.BsmtFullBath)[0][0]
dataset.BsmtHalfBath[dataset.BsmtHalfBath.isnull() == True] = stats.mode(train.BsmtHalfBath)[0][0]
dataset.GarageCars[dataset.GarageCars.isnull() == True] = stats.mode(train.GarageCars)[0][0]
dataset.GarageArea[dataset.GarageArea.isnull() == True] = np.median(train.
 GarageArea[train.GarageArea.isnull() == False])
 dataset.TotalBsmtSF[dataset.TotalBsmtSF.isnull() == True] = np.median(train.
 TotalBsmtSF[train.TotalBsmtSF.isnull() == False])
 dataset.BsmtFinSF1[dataset.BsmtFinSF1.isnull() == True] = np.median(train.
 BsmtFinSF1[train.BsmtFinSF1.isnull() == False])
 dataset.BsmtFinSF2[dataset.BsmtFinSF2.isnull() == True] = np.median(train.
 BsmtFinSF2[train.BsmtFinSF2.isnull() == False])
 dataset.BsmtUnfSF[dataset.BsmtUnfSF.isnull() == True] = np.median(train.
 BsmtUnfSF[train.BsmtUnfSF.isnull() == False])
 dataset.LotFrontage[dataset.LotFrontage.isnull() == True] = np.median(train.
 LotFrontage[train.LotFrontage.isnull() == False] )<filter>
encoder = prepare_pipeline.named_transformers_.nominal.steps[1][1]
print('Before preprocess, number of nominal features: ', len(nominal_features))
print('After preprocess, number of categories:', len(encoder.categories_))
tmp = zip(nominal_features, encoder.categories_)
 all_cats = [feature + '_' + str(suffix)for feature, cat in tmp for suffix in cat]
transformed_features = ordinal_features + discrete_features + continuous_features + all_cats
len(transformed_features )
dataset[dataset.MasVnrArea.isnull() == True].MasVnrType<feature_engineering>
df = pd.DataFrame(train_prepared, columns=transformed_features)
 df['Target'] = train.SalePrice
dataset.MasVnrArea[dataset.MasVnrArea.isnull() == True] = 0<count_missing_values>
x1 = df.YearBuilt > 1890
x2 = df.YearBuilt < 1900
df[x1 & x2].loc[:, ['YearBuilt', 'Target']]
dataset[["GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
"GarageYrBlt"]][dataset.GarageYrBlt.isnull() == True].dropna()<filter>
corr_matrix = df.corr()
corr_matrix.Target.sort_values(ascending=False )
dataset.GarageYrBlt[dataset.GarageYrBlt.isnull() == True] = 'NA'<define_variables>
train_set, test_set = train_test_split(train, test_size = 0.2, shuffle= True, random_state = 42)
 print('train set:', train_set.shape)
 print('test set:', test_set.shape )
num_discrete = ['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd',
'Fireplaces','GarageCars','GarageYrBlt','YearBuilt','YearRemodAdd','YrSold','MoSold']<feature_engineering>
step1_y = np.log(train_set.SalePrice.copy())
 step1_X = prepare_pipeline.fit_transform(train_set )
dataset["Baths"] = dataset.BsmtFullBath + 0.5*dataset.BsmtHalfBath + dataset.FullBath + 0.5*dataset.HalfBath<categorify>
lasso_reg = Lasso()
lasso_reg.fit(step1_X, step1_y)
 pred = lasso_reg.predict(step1_X)
 print(np.sqrt(mean_squared_error(step1_y, pred)))
dataset.Baths = dataset.Baths.apply(lambda x: 3.5 if x > 3.5 else x )<categorify>
eps = 0.0005
alphas = np.logspace(-5 , 1, 100)
 lasso_cv = LassoCV(alphas=alphas, cv = 5, verbose=1, random_state=42)
 lasso_cv.fit(step1_X,step1_y)
 best_alpha = lasso_cv.alpha_
print('best alpha', best_alpha)
 alphas = np.arange(0.95, 1.05, 0.001)* best_alpha
lasso_cv = LassoCV(alphas= alphas, cv = 10, verbose=1, random_state=42)
 lasso_cv.fit(step1_X,step1_y)
 best_alpha = lasso_cv.alpha_
print('refined best alpha', best_alpha )
dataset.BedroomAbvGr = dataset.BedroomAbvGr.apply(lambda x: 5 if x > 5 else x )<data_type_conversions>
scorer = make_scorer(mean_squared_error, greater_is_better = False)
 scores = cross_val_score(Lasso(alpha=best_alpha), step1_X, step1_y, scoring=scorer, cv = 10, verbose=2)
 scores = np.sqrt(-scores)
 print('mean:', scores.mean() , '\t sdv:', scores.std() )
dataset.BedroomAbvGr = dataset.BedroomAbvGr.astype(str )<categorify>
test_X = prepare_pipeline.transform(test_set)
 test_y = np.log(test_set.SalePrice.copy())
 print(step1_X.shape)
 print(test_X.shape)
 lasso_reg = Lasso(alpha=best_alpha)
 lasso_reg.fit(step1_X,step1_y )
dataset.KitchenAbvGr = dataset.KitchenAbvGr.apply(lambda x: 1 if x==0 else(2 if x==3 else x))<data_type_conversions>
pred = lasso_reg.predict(test_X)
 np.sqrt(mean_squared_error(test_y, pred))
dataset.KitchenAbvGr = dataset.KitchenAbvGr.astype(str )<feature_engineering>
all_X = prepare_pipeline.fit_transform(train)
 all_y = np.log(train.SalePrice.copy())
 lasso_reg.fit(all_X, all_y )
dataset.TotRmsAbvGrd = dataset.TotRmsAbvGrd.apply(lambda x: 3 if x==2 else(11 if x>11 else x))<set_options>
final_X = prepare_pipeline.transform(test)
 pred = np.exp(lasso_reg.predict(final_X))
<feature_engineering><EOS> 
result = pd.DataFrame()
result['Id'] = test.Id
result['SalePrice'] = pred
result.to_csv('it2_result.csv', index=False )