Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<predict_on_test>
assert sys.version_info >=(3, 5)
 assert sklearn.__version__ >= "0.20"
%matplotlib inline
warnings.filterwarnings(action="ignore" )
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))<predict_on_test>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train["SalePrice"] = np.log1p(train["SalePrice"])
 print(train.shape, test.shape )
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
def missing_values_calculate(trainset):
nulldata =(trainset.isnull().sum() / len(trainset)) * 100
nulldata = nulldata.drop(nulldata[nulldata == 0].index ).sort_values(ascending=False)
 ratio_missing_data = pd.DataFrame({'Ratio' : nulldata})
 return ratio_missing_data.head(30 )
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
missing_values_calculate(train )
print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.10 +
xgb_train_pred*0.10 + lgb_train_pred*0.80))<define_variables>
def imputer_specific_categories(dataset, listfeatures):
data = dataset.copy()
for names in listfeatures:
data[names] = data[names].fillna("None")
 return data
ensemble = stacked_pred*0.1 + xgb_pred*0.1 + lgb_pred*0.8<save_to_csv>
features_meaning_none_value= ["Alley", "PoolQC", "MiscFeature","Fence","MiscVal","FireplaceQu",
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2','MSSubClass']
train_drop = imputer_specific_categories(train, features_meaning_none_value)
 test = imputer_specific_categories(test,features_meaning_none_value )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<categorify>
missing_values_calculate(train_drop )
def label_encoding(train: pd.DataFrame, test: pd.DataFrame, col_definition: dict):

 n_train = len(train)
 train = pd.concat([train, test], sort=False ).reset_index(drop=True)
 for f in col_definition['encode_col']:
try:
lbl = preprocessing.LabelEncoder()
train[f] = lbl.fit_transform(list(train[f].values))
except:
print(f)
 test = train[n_train:].reset_index(drop=True)
 train = train[:n_train]
return train, test<load_from_csv>
missing_values_calculate(test )
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )<count_unique_values>
train_drop['total_year_build_and_remod']= train_drop['YearBuilt']+train_drop['YearRemodAdd']
train_drop['Total_sqr_are'] =(train_drop['BsmtFinSF1'] + train_drop['BsmtFinSF2'] +
train_drop['1stFlrSF'] + train_drop['2ndFlrSF'])
 train_drop['Total_Bathrooms'] =(train_drop['FullBath'] +(0.5 * train_drop['HalfBath'])+
train_drop['BsmtFullBath'] +(0.5 * train_drop['BsmtHalfBath']))
train_drop['Total_porch_sf'] =(train_drop['OpenPorchSF'] + train_drop['3SsnPorch'] +
train_drop['EnclosedPorch'] + train_drop['ScreenPorch'] +
train_drop['WoodDeckSF'])
 test['total_year_build_and_remod']= test['YearBuilt']+test['YearRemodAdd']
test['Total_sqr_are'] =(test['BsmtFinSF1'] + test['BsmtFinSF2'] +
test['1stFlrSF'] + test['2ndFlrSF'])
 test['Total_Bathrooms'] =(test['FullBath'] +(0.5 * test['HalfBath'])+
test['BsmtFullBath'] +(0.5 * test['BsmtHalfBath']))
test['Total_porch_sf'] =(test['OpenPorchSF'] + test['3SsnPorch'] +
test['EnclosedPorch'] + test['ScreenPorch'] +
test['WoodDeckSF'] )
unique_cols = list(train.columns[train.nunique() == 1])
 print(unique_cols )<count_duplicates>
def has_feature_function(data, column_name_features={}):
for column,name in column_name_features.items() :
data[name] = data[column].apply(lambda x: 1 if x > 0 else 0 )
duplicated_cols = list(train.columns[train.T.duplicated() ])
 print(duplicated_cols )<categorify>
columns = {'PoolArea': 'haspool',
'2ndFlrSF': 'has2ndfloor',
'GarageArea': 'hasgarage',
'TotalBsmtSF': 'hasbasement',
'Fireplaces': 'hasfireplace',
}
train, test = label_encoding(train, test, col_definition={'encode_col':categorical_cols} )<prepare_x_and_y>
train_drop.drop(train_drop[(train_drop["SalePrice"] < 300000)&(train_drop["GrLivArea"] > 4000)].index, inplace=True )
X_train = train.drop(['Id', 'SalePrice'], axis=1)
 y_train = np.log1p(train['SalePrice'])
 X_test = test.drop(['Id', 'SalePrice'], axis=1 )<find_best_model_class>
train_labels = train_drop["SalePrice"].copy()
def objective(trial):
y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params = {
'num_leaves': trial.suggest_int('num_leaves', 20, 30),
'max_depth': trial.suggest_int('max_depth', 3, 6),
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.05
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr,
y_tr,
categorical_feature=categorical_cols)
 lgb_eval = lgb.Dataset(X_val,
y_val,
reference=lgb_train,
categorical_feature=categorical_cols)
 model = lgb.train(params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val,
num_iteration=model.best_iteration)
 y_pred = model.predict(X_test,
num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model)
 score = np.sqrt(mean_squared_error(y_train, oof_train))
return score<find_best_params>
train_drop.drop(["SalePrice",'Utilities', 'Street', 'PoolQC','Id'], axis=1, inplace=True)
 test.drop(['Utilities', 'Street', 'PoolQC','Id'], axis=1, inplace=True )
study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0))
study.optimize(objective, n_trials=40 )<find_best_params>
num_attribs = train_drop.select_dtypes(exclude=['object'])
 cat_attribs = train_drop.select_dtypes(exclude=['int64','float64'] )
study.best_params<drop_column>
num_pipeline = Pipeline([ 
('imputer', SimpleImputer(strategy="median")) , 
('minmaxscaler', MinMaxScaler()),
] )
X_train = X_train.drop(['GrLivArea', 'TotalBsmtSF', 'BsmtFinSF1'], axis=1)
 X_test = X_test.drop(['GrLivArea', 'TotalBsmtSF', 'BsmtFinSF1'], axis=1 )<split>
class MostFrequentImputer(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
self.most_frequent_ = pd.Series([X[c].value_counts().index[0] for c in X],
index=X.columns)
 return self
def transform(self, X, y=None):
return X.fillna(self.most_frequent_ )
y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params = {
'num_leaves': study.best_params['num_leaves'],
'max_depth': study.best_params['max_depth'],
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.05
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr,
y_tr,
categorical_feature=categorical_cols)
 lgb_eval = lgb.Dataset(X_val,
y_val,
reference=lgb_train,
categorical_feature=categorical_cols)
 model = lgb.train(params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val,
num_iteration=model.best_iteration)
 y_pred = model.predict(X_test,
num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model )<compute_test_metric>
cat_pipeline = Pipeline([ 
("imputer", MostFrequentImputer()), 
("cat_encoder", OneHotEncoder()),
] )
print(f'CV: {np.sqrt(mean_squared_error(y_train, oof_train)) }' )<create_dataframe>
full_pipeline = ColumnTransformer([ 
("num", num_pipeline, list(num_attribs)) , 
("cat", cat_pipeline, list(cat_attribs)) ,
] )
importance = pd.DataFrame(model.feature_importance() , index=X_train.columns, columns=['importance'])
 result = importance.sort_values('importance', ascending=False )<save_to_csv>
train_dummies = pd.get_dummies(train_drop)
 test_dummies = pd.get_dummies(test )
y_sub = sum(y_preds)/ len(y_preds)
 y_sub = np.expm1(y_sub)
 sub['SalePrice'] = y_sub
sub.to_csv('submission.csv', index=False)
 sub.head()<load_from_csv>
train_dummies, test_dummies = train_dummies.align(test_dummies, join='inner', axis=1 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
<prepare_x_and_y>
def modelfit(alg, dtrain, target, only_predict = False):
time_start = time.perf_counter()
if not only_predict:
alg.fit(dtrain, target)
 dtrain_predictions = alg.predict(dtrain)
 kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 cv_score = cross_val_score(alg, dtrain,target, cv=kfolds, scoring='neg_mean_squared_error')
 cv_score = np.sqrt(-cv_score)
 time_end = time.perf_counter()
total_time = time_end-time_start
print("
Model Report")
 print("RMSE : {:.4f}".format(np.sqrt(mean_squared_error(target, dtrain_predictions))))
print("CV Score : Mean - %.4f | Std - %.4f | Min - %.4f | Max - %.4f" %(np.mean(cv_score),np.std(cv_score),np.min(cv_score),np.max(cv_score)))
 print("Amount of time spent during training the model and cross validation: %4.3f seconds" %(total_time))
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))<create_dataframe>
train_prepared = full_pipeline.fit_transform(train_drop)
 train_dummies_prepared = num_pipeline.fit_transform(train_dummies )
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
print(len(all_data_na))
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
print(len(all_data_na))
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )<data_type_conversions>
lin_reg = LinearRegression()
modelfit(lin_reg, train_prepared, train_labels )
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )<data_type_conversions>
model_lasso = Lasso()
modelfit(model_lasso, train_dummies_prepared, train_labels )
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )<data_type_conversions>
model_elastic = ElasticNet(max_iter=1e7, l1_ratio=0.9)
 modelfit(model_elastic, train_prepared, train_labels )
all_data["Alley"] = all_data["Alley"].fillna("None" )<data_type_conversions>
tree_reg = DecisionTreeRegressor(random_state=42)
 modelfit(tree_reg, train_prepared, train_labels )
all_data["Fence"] = all_data["Fence"].fillna("None" )<data_type_conversions>
model_kernel = KernelRidge(alpha=0.6, kernel='polynomial', degree=5, coef0=2.5)
 modelfit(model_kernel, train_prepared, train_labels )
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )<categorify>
forest_reg = RandomForestRegressor(n_estimators=40, random_state=42,
min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 modelfit(forest_reg, train_dummies_prepared, train_labels )
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))<data_type_conversions>
tree_ada = DecisionTreeRegressor(random_state = 42,max_depth = 4)
 ada_reg = AdaBoostRegressor(
 tree_reg, n_estimators=3000, random_state=42,learning_rate=0.009, loss='square')
 modelfit(ada_reg, train_dummies_prepared, train_labels )
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
params = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 2,
'learning_rate': 0.01, 'loss': 'ls'}
params_1 = {'n_estimators': 3000, 'learning_rate': 0.05, 'max_depth' : 4,
'max_features': 'sqrt', 'min_samples_leaf': 15, 'min_samples_split': 10,
'loss': 'huber', 'random_state': 42}
train_dummies_prepared = num_pipeline.fit_transform(train_dummies)
 gdb_model = GradientBoostingRegressor(**params_1)
 modelfit(gdb_model, train_dummies_prepared, train_labels )
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
def indices_of_top_k(arr, k):
return np.sort(np.argpartition(np.array(arr), -k)[-k:])
 class TopFeatureSelector(BaseEstimator, TransformerMixin):
def __init__(self, feature_importances, k):
self.feature_importances = feature_importances
self.k = k
def fit(self, X, y=None):
self.feature_indices_ = indices_of_top_k(self.feature_importances, self.k)
 return self
def transform(self, X):
return X[:, self.feature_indices_]
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 6,
'subsample': 0.6,
'colsample_bytree': 0.7,
'eval_metric': 'rmse',
'seed': 42,
'silent': True,
}
model_xgb = xgb.XGBRegressor(**params,n_estimators=3000,learning_rate=0.05,verbose_eval=True )
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
fold_val_pred = []
fold_err = []
fold_importance = []
record = dict()
kfold = list(KFold(10,shuffle = True, random_state = 42 ).split(train_dummies))
def xgb_model(train, params, train_label, fold, verbose, pipeline):
for i,(trn, val)in enumerate(fold):
print(i+1, "fold.RMSE")
 trn_x = train.loc[trn, :]
trn_y = train_label[trn]
val_x = train.loc[val, :]
val_y = train_label[val]
fold_val_pred = []
start = datetime.now()
model = xgb.train(params
, xgb.DMatrix(trn_x, trn_y)
 , 3500
, [(xgb.DMatrix(trn_x, trn_y), 'train'),(xgb.DMatrix(val_x, val_y), 'valid')]
, verbose_eval=verbose
, early_stopping_rounds=500
, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 fold_val_pred.append(val_pred*0.2)
 fold_err.append(record['valid_0']['rmse'][best_idx])
 fold_importance.append(model.feature_importance('gain'))
print("xgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)' )
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )<data_type_conversions>
modelfit(model_xgb, train_dummies, train_labels )
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )<drop_column>
pipe = Pipeline([ 
('pca', PCA(5)) , 
('xgb', xgb.XGBClassifier())
 ])
 param_grid = {
'pca__n_components': [3, 5, 7, 10,15],
'xgb__n_estimators': [500,1000,2000,3500],
'xgb__learning_rate': [0.09,0.1,0.05,0.001]
}
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
lgb_model = lgb.LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.005,
n_estimators=10000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 modelfit(lgb_model, train_dummies, train_labels )
all_data["Functional"] = all_data["Functional"].fillna("Typ" )<data_type_conversions>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )<data_type_conversions>
models_total =(model_kernel, model_lasso, model_elastic, gdb_model, lgb_model)
 model_ada_gbd =(ada_reg,gdb_model)
 stack_gen = StackingCVRegressor(regressors= models_total,
meta_regressor=model_xgb,
use_features_in_secondary=True )
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )<categorify>
modelfit(stack_gen, train_dummies_prepared, train_labels )
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )<data_type_conversions>
test_dummies_prepared = num_pipeline.fit_transform(test_dummies)
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )<data_type_conversions>
prediction_some_regressors = stack_gen.predict(test_dummies_prepared )
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
<sort_values>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['SalePrice'] = np.expm1(prediction_some_regressors)
 sub.to_csv("stack_regression.csv", index=False )