Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
pd.set_option('display.max_columns', 500)
 pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
batch = []
names = []
probs = []
for i in tqdm(range(len(test_dataset))):
sample_spec, _, _, sample_id = test_dataset[i]
if isinstance(sample_spec, list):
batch += sample_spec
else:
batch.append(sample_spec)
 if isinstance(sample_id, list):
names += sample_id
else:
names.append(sample_id)
 if len(batch)>= BATCH_SIZE or i ==(len(test_dataset)- 1):
with torch.no_grad() :
if len(batch)> BATCH_SIZE + 1:
n_steps = math.ceil(len(batch)/ BATCH_SIZE)
 for step in range(n_steps):
small_batch = batch[step*BATCH_SIZE:(step+1)*BATCH_SIZE]
small_batch = torch.stack(small_batch ).to(DEVICE ).float()
batch_probs = sum(m(small_batch ).detach().cpu() for m in models)/ len(models)
 batch_probs = batch_probs.numpy()
probs.append(batch_probs)
 else:
batch = torch.stack(batch ).to(DEVICE ).float()
batch_probs = sum(m(batch ).detach().cpu() for m in models)/ len(models)
 batch_probs = batch_probs.numpy()
probs.append(batch_probs)
 batch = []<create_dataframe>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 train.head(5 )
probs = np.concatenate(probs)
 result_df = pd.DataFrame({
'row_id':names,
'birds':[probs[i] for i in range(probs.shape[0])]
})
 result_df = result_df.groupby('row_id')['birds'].apply(lambda x: np.stack(x ).max(0)).reset_index()
result_df<categorify>
train_id = train["Id"]
test_id = test["Id"]
train.drop("Id", axis=1, inplace=True)
 test.drop("Id", axis=1, inplace=True)
 train.drop(train[(train['GrLivArea'] > 4000)&(train['SalePrice'] < 300000)].index, axis=0, inplace=True)
 train['SalePrice'] = np.log1p(train['SalePrice'])
 ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train['SalePrice'].values
all_data = pd.concat([train, test] ).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 all_data["PoolQC"] = all_data["PoolQC"].fillna("None")
 all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None")
 all_data["Alley"] = all_data["Alley"].fillna("None")
 all_data["Fence"] = all_data["Fence"].fillna("None")
 all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None")
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None')
 all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0)
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data.drop(['Utilities'], axis=1, inplace=True)
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
 all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
 cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for col in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[col].values))
all_data[col] = lbl.transform(list(all_data[col].values))
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness = skewness[abs(skewness)> 0.75]
skewed_features = skewness.index
lam = 0.15
for feature in skewed_features:
all_data[feature] = boxcox1p(all_data[feature], lam)
 all_data = pd.get_dummies(all_data)
 train = all_data[:ntrain]
test = all_data[ntrain:]
def probs2names(
 probs_array: np.ndarray,
threshold: int = 0.5,
max_birds_to_take: Optional[int] = None,
idx2bird_mapping: Mapping[int, str] = CODE2BIRD 
):
accepted_indices = np.where(probs_array > threshold)[0]
if max_birds_to_take is not None:
accepted_probs = probs_array[accepted_indices]
accepted_probs_indices = np.argsort(-accepted_probs)
 accepted_indices = accepted_indices[accepted_probs_indices]
accepted_indices = accepted_indices[:max_birds_to_take]
if len(accepted_indices)== 0:
return 'nocall'
else:
return ' '.join([idx2bird_mapping[idx] for idx in accepted_indices] )<feature_engineering>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=1 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, random_state =7,
verbosity=0, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 models = [('Lasso', lasso),('ElasticNet', ENet),('Kernel Ridge', KRR), 
('Gradient Boosting', GBoost),('Xgboost', model_xgb),('LGBM', model_lgb)]
for model in models:
score = rmsle_cv(model[1])
 print('{} score: {}({})'.format(model[0], score.mean() , score.std()))
result_df['birds'] = result_df['birds'].apply(lambda x: probs2names(
 x,
threshold=SIGMOID_THRESH,
max_birds_to_take=MAX_BIRDS 
))<merge>
class AveragingModel(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, x, y):
self.models_ = [clone(model)for model in self.models]
for model in self.models_:
model.fit(x, y)
 return self
def predict(self, x):
predictions = np.column_stack([
model.predict(x)for model in self.models_
])
 return np.mean(predictions, axis=1)
 averaging_model = AveragingModel(models=[ENet, GBoost, KRR, lasso])
 score = rmsle_cv(averaging_model)
 print("Averaging model score: {:.4f}({:.4f})".format(score.mean() , score.std()))
class StackingModel(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, x, y):
self.base_models_ = [list() for model in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=1)
 out_of_fold_predictions = np.zeros(( x.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(x, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(x[train_index], y[train_index])
 y_pred = instance.predict(x[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, x):
meta_features = np.column_stack([
np.column_stack([model.predict(x)for model in trained_base_models] ).mean(axis=1)
 for trained_base_models in self.base_models_])
 return self.meta_model_.predict(meta_features)
 stacking_model = StackingModel(base_models=[ENet, GBoost, KRR], meta_model=lasso)
 score = rmsle_cv(stacking_model)
 print("Stacking model score: {:.4f}({:.4f})".format(score.mean() , score.std()))
<count_values><EOS> 
for model in [stacking_model, model_xgb, model_lgb]:
model.fit(train.values, y_train)
 stacking_pred = np.expm1(stacking_model.predict(test.values))
xgb_pred = np.expm1(model_xgb.predict(test.values))
lgb_pred = np.expm1(model_lgb.predict(test.values))
ensemble = stacking_pred * 0.7 + xgb_pred * 0.15 + lgb_pred * 0.15
submission = pd.DataFrame({'Id': test_id, 'SalePrice': ensemble})
 submission.to_csv('submission.csv', index=False )