Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<normalization>
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 128
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(1024, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<choose_model_class>
train.drop(train[train["GrLivArea"] > 4000].index, inplace=True)
 test.loc[1116, "GarageType"] = np.nan
train["SalePrice"] = np.log1p(train["SalePrice"])
 y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True ).drop(['SalePrice'], axis=1 )
m = NeuralNet()<train_model>
def fillNA(data, var, fill=None, custom=False):
if not fill:
data_map = {
"float64": 0,
"int64": 0,
"object": "None"
}
data[var] = data[var].fillna(data_map[str(data[var].dtype)])
 else:
if custom:
data[var] = data[var].fillna(fill)
 elif fill == 'mode':
data[var] = data[var].fillna(data[var].mode() [0])
 return data
impute_vals = ["BedroomAbvGr", "Alley", "GarageYrBlt", "GarageArea", "GarageCars", 'BsmtFinSF1', 'PoolArea', 'MSSubClass',
'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'HalfBath', 'KitchenAbvGr', 'WoodDeckSF',
'BsmtHalfBath', 'MasVnrType', 'MasVnrArea', 'MiscVal', 'OpenPorchSF', 'ScreenPorch', 'TotRmsAbvGrd', "EnclosedPorch"]
for var in impute_vals:
all_data = fillNA(all_data, var)
 impute_mode = ['MSZoning', 'Electrical', 'Exterior1st', 'Exterior2nd', 'SaleType']
for var in impute_mode:
all_data = fillNA(all_data, var, fill='mode')
 impute_custom = {
"CentralAir": "N",
"Condition1": "Norm",
"Condition1": "Norm",
"Functional": "Typ",
"HeatingQC" : "TA",
"LotShape" : "Reg",
"PavedDrive": "N",
"PoolQC" : "No",
"BsmtQual" : "No",
"BsmtCond" : "No",
"BsmtExposure" : "No",
"BsmtFinType1" : "No",
"BsmtFinType2" : "No",
"ExterCond" : "No",
"ExterQual" : "No",
"Fence" : "No",
"FireplaceQu" : "No",
"GarageType" : "No",
"GarageFinish" : "No",
"GarageQual" : "No",
"GarageCond" : "No",
"KitchenQual" : "TA",
"MiscFeature" : "No",
"PavedDrive" : "N",
"SaleCondition": "Normal"
}
for var in impute_custom:
all_data = fillNA(all_data, var, impute_custom[var], custom=True)
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data = all_data.drop(['Utilities'], axis=1)
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
def train_model(model, x_train, y_train, x_val, y_val, validate=True):
optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train, y_train)
 valid = torch.utils.data.TensorDataset(x_val, y_val)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean' ).cuda()
best_score = -np.inf
for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( x_val_fold.size(0)))
 if validate:
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
search_result = threshold_search(y_val.cpu().numpy() , valid_preds)
 val_f1, val_threshold = search_result['f1'], search_result['threshold']
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} best_t={:.2f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))
else:
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
valid_preds = np.zeros(( x_val_fold.size(0)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
print('Validation loss: ', avg_val_loss)
 test_preds = np.zeros(( len(test_loader.dataset)))
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
return valid_preds, test_preds<create_dataframe>
for var in ['MSSubClass', 'YrSold', 'OverallCond', 'MoSold']:
all_data[var] = all_data[var].astype(str)
 all_data = all_data.replace({"Alley" : {"No" : 0, "Grvl" : 1, "Pave" : 2},
"BsmtCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"No" : 0, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"Functional" : {"No" : 0, "Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5, "Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PavedDrive" : {"N" : 0, "P" : 1, "Y" : 2},
"PoolQC" : {"No" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Street" : {"Grvl" : 1, "Pave" : 2},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}} 
)
 all_data["SimplOverallQual"] = all_data.OverallQual.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2,7 : 3, 8 : 3, 9 : 3, 10 : 3})
 all_data["SimplOverallCond"] = all_data.OverallCond.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2,7 : 3, 8 : 3, 9 : 3, 10 : 3})
 all_data["SimplPoolQC"] = all_data.PoolQC.replace({1 : 1, 2 : 1,3 : 2, 4 : 2})
 all_data["SimplGarageCond"] = all_data.GarageCond.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplGarageQual"] = all_data.GarageQual.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplFireplaceQu"] = all_data.FireplaceQu.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplFireplaceQu"] = all_data.FireplaceQu.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplFunctional"] = all_data.Functional.replace({1 : 1, 2 : 1,3 : 2, 4 : 2,5 : 3, 6 : 3, 7 : 3, 8 : 4})
 all_data["SimplKitchenQual"] = all_data.KitchenQual.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplHeatingQC"] = all_data.HeatingQC.replace({1 : 1,4 : 2, 5 : 2})
 all_data["SimplBsmtFinType1"] = all_data.BsmtFinType1.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2 })
 all_data["SimplBsmtFinType2"] = all_data.BsmtFinType2.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2})
 all_data["SimplBsmtCond"] = all_data.BsmtCond.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplBsmtQual"] = all_data.BsmtQual.replace({1 : 1,4 : 2, 5 : 2})
 all_data["SimplExterCond"] = all_data.ExterCond.replace({1 : 1,4 : 2, 5 : 2})
 all_data["SimplExterQual"] = all_data.ExterQual.replace({1 : 1,4 : 2, 5 : 2})
 all_data["OverallGrade"] = all_data["OverallQual"] * all_data["OverallCond"]
all_data["GarageGrade"] = all_data["GarageQual"] * all_data["GarageCond"]
all_data["ExterGrade"] = all_data["ExterQual"] * all_data["ExterCond"]
all_data["KitchenScore"] = all_data["KitchenAbvGr"] * all_data["KitchenQual"]
all_data["FireplaceScore"] = all_data["Fireplaces"] * all_data["FireplaceQu"]
all_data["GarageScore"] = all_data["GarageArea"] * all_data["GarageQual"]
all_data["PoolScore"] = all_data["PoolArea"] * all_data["PoolQC"]
all_data["SimplOverallGrade"] = all_data["SimplOverallQual"] * all_data["SimplOverallCond"]
all_data["SimplExterGrade"] = all_data["SimplExterQual"] * all_data["SimplExterCond"]
all_data["SimplPoolScore"] = all_data["PoolArea"] * all_data["SimplPoolQC"]
all_data["SimplGarageScore"] = all_data["GarageArea"] * all_data["SimplGarageQual"]
all_data["SimplFireplaceScore"] = all_data["Fireplaces"] * all_data["SimplFireplaceQu"]
all_data["SimplKitchenScore"] = all_data["KitchenAbvGr"] * all_data["SimplKitchenQual"]
all_data["TotalBath"] = all_data["BsmtFullBath"] +(0.5 * all_data["BsmtHalfBath"])+ \
all_data["FullBath"] +(0.5 * all_data["HalfBath"])
 all_data["AllSF"] = all_data["GrLivArea"] + all_data["TotalBsmtSF"]
all_data["AllFlrsSF"] = all_data["1stFlrSF"] + all_data["2ndFlrSF"]
all_data["AllPorchSF"] = all_data["OpenPorchSF"] + all_data["EnclosedPorch"] + \
all_data["3SsnPorch"] + all_data["ScreenPorch"]
all_data["HasMasVnr"] = all_data.MasVnrType.replace({"BrkCmn" : 1, "BrkFace" : 1, "CBlock" : 1, "Stone" : 1, "None" : 0})
 all_data["BoughtOffPlan"] = all_data.SaleCondition.replace({"Abnorml" : 0, "Alloca" : 0, "AdjLand" : 0, "Family" : 0, "Normal" : 0, "Partial" : 1})
 all_data["OverallQual-s2"] = all_data["OverallQual"] ** 2
all_data["OverallQual-s3"] = all_data["OverallQual"] ** 3
all_data["OverallQual-Sq"] = np.sqrt(all_data["OverallQual"])
 all_data["AllSF-2"] = all_data["AllSF"] ** 2
all_data["AllSF-3"] = all_data["AllSF"] ** 3
all_data["AllSF-Sq"] = np.sqrt(all_data["AllSF"])
 all_data["AllFlrsSF-2"] = all_data["AllFlrsSF"] ** 2
all_data["AllFlrsSF-3"] = all_data["AllFlrsSF"] ** 3
all_data["AllFlrsSF-Sq"] = np.sqrt(all_data["AllFlrsSF"])
 all_data["GrLivArea-2"] = all_data["GrLivArea"] ** 2
all_data["GrLivArea-3"] = all_data["GrLivArea"] ** 3
all_data["GrLivArea-Sq"] = np.sqrt(all_data["GrLivArea"])
 all_data["SimplOverallQual-s2"] = all_data["SimplOverallQual"] ** 2
all_data["SimplOverallQual-s3"] = all_data["SimplOverallQual"] ** 3
all_data["SimplOverallQual-Sq"] = np.sqrt(all_data["SimplOverallQual"])
 all_data["ExterQual-2"] = all_data["ExterQual"] ** 2
all_data["ExterQual-3"] = all_data["ExterQual"] ** 3
all_data["ExterQual-Sq"] = np.sqrt(all_data["ExterQual"])
 all_data["GarageCars-2"] = all_data["GarageCars"] ** 2
all_data["GarageCars-3"] = all_data["GarageCars"] ** 3
all_data["GarageCars-Sq"] = np.sqrt(all_data["GarageCars"])
 all_data["TotalBath-2"] = all_data["TotalBath"] ** 2
all_data["TotalBath-3"] = all_data["TotalBath"] ** 3
all_data["TotalBath-Sq"] = np.sqrt(all_data["TotalBath"])
 all_data["KitchenQual-2"] = all_data["KitchenQual"] ** 2
all_data["KitchenQual-3"] = all_data["KitchenQual"] ** 3
all_data["KitchenQual-Sq"] = np.sqrt(all_data["KitchenQual"])
 all_data["GarageScore-2"] = all_data["GarageScore"] ** 2
all_data["GarageScore-3"] = all_data["GarageScore"] ** 3
all_data["GarageScore-Sq"] = np.sqrt(all_data["GarageScore"])
 all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
categorical_features = all_data.select_dtypes(include = ["object"] ).columns
numerical_features = all_data.select_dtypes(exclude = ["object"] ).columns
train_num = all_data[numerical_features]
train_cat = all_data[categorical_features]
skewness = train_num.apply(lambda x: skew(x))
skewness = skewness[abs(skewness)> 0.75]
skewed_features = skewness.index
train_num[skewed_features] = boxcox1p(train_num[skewed_features], 0.15)
 train_cat = pd.get_dummies(train_cat)
 all_data = pd.concat([train_num, train_cat], axis = 1)
 train = all_data[:train.shape[0]]
test = all_data[train.shape[0]:]
scale = StandardScaler()
train.loc[:, numerical_features] = scale.fit_transform(train.loc[:, numerical_features])
 test.loc[:, numerical_features] = scale.transform(test.loc[:, numerical_features])
 drop_cols = ["Exterior1st_ImStucc", "Exterior1st_Stone", "Exterior2nd_Other","HouseStyle_2.5Fin",
"RoofMatl_Membran", "RoofMatl_Metal", "RoofMatl_Roll",
"Condition2_RRAe", "Condition2_RRAn", "Condition2_RRNn",
"Heating_Floor", "Heating_OthW", "MSSubClass_150",
"Electrical_Mix", "MiscFeature_TenC", "Condition2_PosN", "MSZoning_C(all)", "MSSubClass_160"
]
train.drop(drop_cols, axis=1, inplace=True)
 test.drop(drop_cols, axis=1, inplace=True)
 print(train.shape, test.shape )
x_test_cuda = torch.tensor(X_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 batch_size = 512
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<compute_train_metric>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 print(train.shape, y_train.shape)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.2, gamma=0,
learning_rate=0.01, max_depth=4,
min_child_weight=1.5, n_estimators=7200,
reg_alpha=0.9, reg_lambda=0.6,
subsample=0.2, silent=1,
seed = 25)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
<prepare_x_and_y><EOS> 
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR, model_xgb, model_lgb), meta_model = lasso)
 score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
averaged_models.fit(train.values, y_train)
 stacked_pred = np.expm1(averaged_models.predict(test.values))
final_prediction = pd.DataFrame({"Id": test_ID, "SalePrice": stacked_pred})
 final_prediction.to_csv('submission.csv',index=False )