Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42
print(train_data_ac_hv_co <(Q1 - 1.5 * IQR)) or(train_data_ac_hv_co >(Q3 + 1.5 * IQR))<filter>
train=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
train_data_ol=train_data_ac_hv_co.copy()
train_data_out = train_data_ol[(( train_data_ol >=(Q1 - 1.5 * IQR)) &(train_data_ol <=(Q3 + 1.5 * IQR)) ).all(axis=1)]<count_values>
df_all = train.append(test, ignore_index=True)
 train_idx = len(train)
 test_idx = len(df_all)- len(test)
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 df_all.drop("Id", axis = 1, inplace = True)
 print("
The train data size after dropping Id feature is : {} ".format(train.shape))
print("The test data size after dropping Id feature is : {} ".format(test.shape))
train_data_out["TARGET"].value_counts()<feature_engineering>
def missing_percentage(df):

 total = df.isnull().sum().sort_values(ascending = False)[df.isnull().sum().sort_values(ascending = False)!= 0]
percent = round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)[round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)!= 0]
return pd.concat([total, percent], axis=1, keys=['Total','Percent'])
 missing_percentage(df_all )
train_data.var3 = train_data.var3.replace(-999999,2 )<feature_engineering>
df_all['MSZoning'] = df_all.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
df_all["LotFrontage"] = df_all.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
test_data.var3 = test_data.var3.replace(-999999,2 )<feature_engineering>
for col in(
 'PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'MasVnrType',
'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2' 
):
df_all[col] = df_all[col].fillna("None" )
test_data_hv_co.var3 = test_data_hv_co.var3.replace(-999999,2)
 train_data_hv_co.var3 = train_data_hv_co.var3.replace(-999999,2 )<categorify>
for col in(
 'LotFrontage', 'MasVnrArea',
'GarageYrBlt', 'GarageArea', 'GarageCars',
'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 
):
df_all[col] = df_all[col].fillna(0 )
test_data_ac.var3 = test_data_ac.var3.replace(-999999,2)
 train_data_ac.var3 = train_data_ac.var3.replace(-999999,2 )<feature_engineering>
df_all['Electrical'] = df_all['Electrical'].fillna(df_all['Electrical'].mode() [0])
 df_all['KitchenQual'] = df_all['KitchenQual'].fillna(df_all['KitchenQual'].mode() [0])
 df_all['Exterior1st'] = df_all['Exterior1st'].fillna(df_all['Exterior1st'].mode() [0])
 df_all['Exterior2nd'] = df_all['Exterior2nd'].fillna(df_all['Exterior2nd'].mode() [0])
 df_all['SaleType'] = df_all['SaleType'].fillna(df_all['SaleType'].mode() [0])
 df_all['Functional'] = df_all['Functional'].fillna(df_all['Functional'].mode() [0] )
test_data_ac_hv.var3 = test_data_ac_hv.var3.replace(-999999,2)
 train_data_ac_hv.var3 = train_data_ac_hv.var3.replace(-999999,2 )<categorify>
df_all = df_all.drop(['Utilities'], axis=1 )
test_data_ac_hv_co.var3 = test_data_ac_hv_co.var3.replace(-999999,2)
 train_data_ac_hv_co.var3 = train_data_ac_hv_co.var3.replace(-999999,2 )<drop_column>
missing_percentage(df_all )
train_data_scaled2= train_data_hv_co.drop(["ID","TARGET"],axis=1)
 train_data_scaled1= train_data_ac_hv_co.drop(["ID","TARGET"],axis=1)
 train_data_scaled= train_data.drop(["ID","TARGET"],axis=1 )<normalization>
df_all['TotalPorchArea'] = df_all['OpenPorchSF'] + df_all['EnclosedPorch'] + df_all['3SsnPorch'] + df_all['ScreenPorch']
df_all['TotalSF'] = df_all['TotalBsmtSF'] + df_all['GrLivArea']
df_all['TotalBath'] = df_all['BsmtFullBath'] + df_all['BsmtHalfBath'] + df_all['FullBath'] + df_all['HalfBath']
def garage_age(row):
if row['GarageYrBlt'] == 0:
return 0
return(row['YrSold'] - row['GarageYrBlt'])
 df_all['GarageAge'] = df_all.apply(garage_age, axis=1)
 df_all['haspool'] = df_all['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all['has2ndfloor'] = df_all['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all['hasgarage'] = df_all['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all['hasbsmt'] = df_all['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all['hasfireplace'] = df_all['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
for data in [train_data_scaled,train_data_scaled1,train_data_scaled2]:
scaler = StandardScaler()
find_pca_components(scaler.fit_transform(data))<compute_test_metric>
df_all['HouseAge'] = df_all['YrSold'] - df_all['YearBuilt']
def remodelled(row):
if row['YearRemodAdd'] > row['YearBuilt']:
return 1
return 0
df_all['Remodelled'] = df_all.apply(remodelled, axis=1)
 def last_modelled(row):
year = max(row['YearBuilt'], row['YearRemodAdd'])
 return row['YrSold'] - year
df_all['LastModelled'] = df_all.apply(last_modelled, axis=1 )
def pca_analysis(n_co,data):
pca = PCA(n_components=n_co)
 data_transformed = pca.fit_transform(data)
 print("PCA Analysis for %s pca components"%(n_co))
print("Eigen vector for each principal component : ",pca.components_)
 print("Amount of variance by each PCA : ", pca.explained_variance_)
 print("Percentage of variance by each PCA : ", pca.explained_variance_ratio_)
 print("number of features in training data : ", pca.n_features_)
 print("number of samples in training data: ", pca.n_samples_)
 print("noise variance of the data : ",pca.noise_variance_)
 return pd.DataFrame(data_transformed )<normalization>
df_all['MSSubClass'] = df_all['MSSubClass'].astype(str)
 df_all['YrSold'] = df_all['YrSold'].astype(str)
 df_all['MoSold'] = df_all['MoSold'].astype(str )
data_transformed=[]
for no, data in [(48,train_data_scaled),(44,train_data_scaled1),(60,train_data_scaled2)]:
scaler = StandardScaler()
data_transformed.append(pca_analysis(no,scaler.fit_transform(data)) )<categorify>
label_features = ['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold']
for feature in label_features:
df_all[feature] = LabelEncoder().fit_transform(df_all[feature] )
def upsampling_dataset(data):
data_majority=data[data.TARGET==0]
data_minority=data[data.TARGET==1]
data_minority_upsampled=resample(data_minority,replace=True,n_samples=73012)
 data_upsampled=pd.concat([data_minority_upsampled,data_majority])
 data_upsampled.info()
print(data_upsampled['TARGET'].value_counts())
 return data_upsampled<define_variables>
cat_features = []
for i in df_all.columns:
if df_all[i].dtype == "object":
if i not in label_features:
cat_features.append(i)
 encoded_features = []
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df_all[feature].values.reshape(-1, 1)).toarray()
n = df_all[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df_all.index
encoded_features.append(encoded_df)
 len_cat = len(cat_features)
 df_all = pd.concat([df_all, *encoded_features[:len_cat]], axis=1 )
datasets = [train_data_ac,train_data_ac_hv_co,train_data_hv_co]<define_variables>
df_all.drop(cat_features, axis=1, inplace=True )
test_datasets = [test_data_ac,test_data_ac_hv_co,test_data_hv_co]<categorify>
X = df_all[:train_idx].drop(['SalePrice'], axis=1)
 y = df_all['SalePrice'][:train_idx]
X_test = df_all[train_idx:].drop(['SalePrice'], axis=1 )
upsampled_data = []
for data in datasets:
upsampled_data.append(upsampling_dataset(data))<prepare_x_and_y>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
if 'ID' not in data_transformed[2]:
data_transformed[2].insert(1,'ID',train_data['ID'])
 x_pca = data_transformed[2]
y_pca = train_data["TARGET"]<prepare_x_and_y>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, X.values, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
x_upsample = upsampled_data[2].drop("TARGET",axis=1)
 y_upsample = upsampled_data[2]["TARGET"]<prepare_x_and_y>
transformer = RobustScaler().fit(X)
 Xt = transformer.transform(X)
 Xt_test = transformer.transform(X_test )
x = datasets[2].drop("TARGET",axis=1)
 y = datasets[2]["TARGET"]<split>
max_evals = 10
space={
'alpha': hp.uniform("alpha", 0.0001, 0.1),
'max_iter': hp.uniform("max_iter", 200, 2000),
}
def hyperparameter_tuning(space):
model=Lasso(
 alpha =(space['alpha']),
max_iter =int(space['max_iter']),
random_state = 1 
)
 scores = cross_val_score(model, Xt, y, scoring='neg_mean_squared_error', cv=KFold(n_splits=5,shuffle=True))
print("SCORE: %0.4f" %(scores.mean()))
return {'loss':-scores.mean() , 'status': STATUS_OK }
trials = Trials()
best = fmin(fn=hyperparameter_tuning,
space=space,
algo=tpe.suggest,
max_evals=max_evals,
trials=trials)
 print(best )
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.30, random_state=44)
 X_pca_train, X_pca_test, y_pca_train, y_pca_test = train_test_split(x_pca, y_pca, test_size=0.30, random_state=44)
 X_upsample_train, X_upsample_test, y_upsample_train, y_upsample_test = train_test_split(x_upsample, y_upsample, test_size=0.30, random_state=44 )<import_modules>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel<train_model>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
sel_ = SelectFromModel(RandomForestClassifier(n_estimators=200))
sel_.fit(X_train.fillna(0), y_train )<features_selection>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
selected_features = X_train.columns[(sel_.get_support())]
len(selected_features )<filter>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213,
random_state =7, nthread = -1)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
train_data[selected_features]<import_modules>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier, BaggingClassifier,GradientBoostingClassifier
from xgboost import XGBClassifier
from sklearn.svm import SVC<choose_model_class>
clf = RandomForestRegressor(n_estimators = 100, 
)
 scores = cross_val_score(clf, X, y, cv=5, scoring='neg_root_mean_squared_error')
 print(scores.mean() )
models = []
models.append(( 'LR', LogisticRegression(class_weight='balanced')))
 models.append(( 'Bagging Classifier',BaggingClassifier()))
models.append(( 'KNN', KNeighborsClassifier(weights='distance')))
 models.append(( 'RandomForest', RandomForestClassifier(class_weight='balanced')))
 models.append(( 'DecisionTree', DecisionTreeClassifier(class_weight='balanced')))
 models.append(( 'GradientBoosting', GradientBoostingClassifier()))
models.append(( 'xgb', XGBClassifier(missing=np.nan, max_depth=6,
n_estimators=350, learning_rate=0.025, nthread=4, subsample=0.95,
colsample_bytree=0.85, seed=4242)) )<import_modules>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
from sklearn import metrics
from sklearn.metrics import classification_report,confusion_matrix
from sklearn.model_selection import cross_val_score
from sklearn.metrics import f1_score,roc_curve,roc_auc_score,precision_score,recall_score,accuracy_score<compute_train_metric>
averaged_models = AveragingModels(models =(
 lasso,
ENet,
GBoost,
model_lgb,
model_xgb 
))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def model_analysis(title,x_train,y_train,x_test,y_test):
df_scores=pd.DataFrame()
pred_dict={}
for name,model in models:
model.fit(x_train,y_train)
 y_pred = model.predict(x_test)
 pred_dict[name] = y_pred
confusion = confusion_matrix(y_test,y_pred)
 TP = confusion[1, 1]
TN = confusion[0, 0]
FP = confusion[0, 1]
FN = confusion[1, 0]
accuracy = metrics.accuracy_score(y_test, y_pred)
 error = 1-accuracy
sensitivity = TP / float(FN + TP)
 specificity = TN /(TN + FP)
 False_positive_rate = 1-specificity
precision = TP / float(TP + FP)
 bal_acc = metrics.balanced_accuracy_score(y_test, y_pred)
 Null_accuracy = max(y_test.mean() ,(1 - y_test.mean()))
f1 = metrics.f1_score(y_test,y_pred)
 auc_score = metrics.roc_auc_score(y_test,y_pred)
 clf_score = pd.DataFrame(
 {name: [accuracy, bal_acc, Null_accuracy,precision,sensitivity,f1,error,specificity,auc_score]},
index=['Accuracy', 'Balanced accuracy','Null_accuracy','precision','recall','f1 score','error','specificity','auc_score'] 
)
 df_scores = pd.concat([df_scores, clf_score], axis=1 ).round(decimals=3)
 print("Roc_curve for all models")
 plot_roc_curve(y_test,pred_dict)
 print(title,end='
')
 print(df_scores.to_markdown() ,end='
')
 model_comparison_plot(df_scores )<compute_test_metric>
averaged_models.fit(X, y)
 av_prices = np.exp(averaged_models.predict(X_test))
model_analysis("Model with normal data",X_train,y_train,X_test,y_test )<compute_test_metric>
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = av_prices
os.chdir('/kaggle/working')
 sub.to_csv('av_submission.csv',index=False )
model_analysis("Model with upsampled data",X_upsample_train,y_upsample_train,X_upsample_test,y_upsample_test )<compute_test_metric>
X_train, X_t, y_train, y_t = train_test_split(X, y, test_size=0.33, random_state=42 )
model_analysis("Model with pca data",X_pca_train,y_pca_train,X_pca_test,y_pca_test )<choose_model_class>
lasso_b = lasso.fit(X_train, y_train)
 enet_b = ENet.fit(X_train, y_train)
 gb_b = GBoost.fit(X_train, y_train)
 xgb_b = model_xgb.fit(X_train, y_train)
 lgb_b = model_lgb.fit(X_train, y_train )
final_model = RandomForestClassifier(class_weight='balanced',random_state=42 )<train_model>
y_pred_lasso = pd.DataFrame(lasso_b.predict(X_t))
y_pred_enet = pd.DataFrame(enet_b.predict(X_t))
y_pred_gb = pd.DataFrame(gb_b.predict(X_t))
y_pred_xgb = pd.DataFrame(xgb_b.predict(X_t))
y_pred_lgb = pd.DataFrame(lgb_b.predict(X_t))
final_model.fit(X_upsample_train,y_upsample_train )<predict_on_test>
y_pred=pd.DataFrame()
best_score = 100
blend_increments = [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]
best_increments = []
for i in blend_increments:
for j in blend_increments:
for k in blend_increments:
for l in blend_increments:
for m in blend_increments:
if(i+j+k+l)> 0:
y_pred['SalePrice'] =(i * y_pred_lasso[0] + j * y_pred_enet[0] + k * y_pred_gb[0] + l * y_pred_xgb[0] + m * y_pred_lgb[0])
 y_pred['SalePrice'] = y_pred['SalePrice'] /(i+j+k+l+m)
 rms = np.sqrt(mean_squared_error(y_pred['SalePrice'], y_t))
if rms < best_score:
best_score = rms
best_increments = [i, j, k, l, m]
print(best_score)
 print(best_increments)
probs = final_model.predict_proba(test_data_hv_co )<create_dataframe>
lasso_b = lasso.fit(X, y)
 enet_b = ENet.fit(X, y)
 gb_b = GBoost.fit(X, y)
 xgb_b = model_xgb.fit(X, y)
 lgb_b = model_lgb.fit(X, y )
submission = pd.DataFrame({"ID":test_data_hv_co.ID, "TARGET": probs[:,1]} )<save_to_csv>
X_test = df_all[train_idx:].drop(['SalePrice'], axis=1)
 y_pred_lasso = pd.DataFrame(lasso_b.predict(X_test))
y_pred_enet = pd.DataFrame(enet_b.predict(X_test))
y_pred_gb = pd.DataFrame(gb_b.predict(X_test))
y_pred_xgb = pd.DataFrame(xgb_b.predict(X_test))
y_pred_lgb = pd.DataFrame(lgb_b.predict(X_test))
submission.to_csv("santander_solution.csv" )<train_model>
y_pred = pd.DataFrame()
xgb_classifier = XGBClassifier(missing=np.nan, max_depth=6,
n_estimators=350, learning_rate=0.025, nthread=4, subsample=0.95,
colsample_bytree=0.85, seed=4242)
 xgb_mdl = CalibratedClassifierCV(xgb_classifier, method='isotonic', cv=10)
 xgb_mdl.fit(X_upsample_train,y_upsample_train )<predict_on_test>
y_pred['SalePrice'] = np.exp(( 0.476 * y_pred_lasso[0] + 0 * y_pred_enet[0] + 0.381 * y_pred_gb[0] + 0.143 * y_pred_xgb[0] + 0 * y_pred_lgb[0]))
probs_xgb = xgb_mdl.predict_proba(test_data_hv_co )<create_dataframe>
y_pred['Id'] = test_ID
submission1 = pd.DataFrame({"ID":test_data_hv_co.ID, "TARGET": probs_xgb[:,1]} )<save_to_csv>
os.chdir('/kaggle/working')
 y_pred.to_csv('blend_submission.csv',index=False )
submission1.to_csv("submission_xgb.csv",index=False )<save_to_csv>
estimators = [ 
('lasso', lasso), 
('ENet', ENet), 
('model_gb', GBoost), 
('model_lgb', model_lgb), 
('model_xgb', model_xgb),
]
reg = StackingRegressor(
 estimators=estimators,
final_estimator=model_lgb,
passthrough = True 
)
 score = rmsle_cv(reg)
 print("Ensemble score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
submission1.to_csv("/kaggle/working/submission.csv",index=False )<set_options>
reg.fit(X, y)
 stack_prices = np.exp(reg.predict(X_test))
<load_from_csv><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = stack_prices
os.chdir('/kaggle/working')
 sub.to_csv('stack_submission.csv',index=False )