Kaggle competition: House Prices - Advanced Regression Techniques
data = data.merge(orders_future, on='user_id', how='left')
 data.head()<split>
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
data_train = data[data.eval_set == 1]
data_train.head()<merge>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
data_train = data_train.merge(order_products_train[['product_id', 'order_id', 'reordered']], on=['product_id', 'order_id'], how='left')
 data_train.head()<drop_column>
train.isna().sum().sort_values(ascending=False )
data_train.drop(['eval_set', 'order_id'], axis=1, inplace=True )<split>
train.isna().sum().sort_values(ascending=False)[:20]
data_test = data[data.eval_set == 2]
data_test.head()<drop_column>
( train.isna().sum() / train.shape[0] ).sort_values(ascending=False)[:20]
data_test.drop(['eval_set', 'order_id'], axis=1, inplace=True )<merge>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
products = pd.read_csv(PATH + 'products.csv', usecols=['product_id', 'aisle_id', 'department_id'])
 data_train = data_train.merge(products, on='product_id', how='left')
 data_test = data_test.merge(products, on='product_id', how='left')
 data_train = data_train.set_index(['user_id', 'product_id'])
 data_test = data_test.set_index(['user_id', 'product_id'] )<categorify>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
columns_mean = ['aisle_id', 'department_id']
for col in columns_mean:
mean = data_train.groupby(col ).reordered.mean()
data_train[col] = data_train[col].map(mean)
 data_test[col] = data_test[col].map(mean )<drop_column>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
del [data, orders_future, products, order_products_train]
gc.collect()<import_modules>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.metrics import f1_score, classification_report
from scikitplot.metrics import plot_confusion_matrix
from scikitplot.classifiers import plot_feature_importances
import xgboost as xgb
import lightgbm as lgb<split>
all_data["Alley"] = all_data["Alley"].fillna("None" )
X = data_train.drop(['reordered', 'uxp_ratio_last_five'], axis=1)
 y = data_train.reordered
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=10 )<drop_column>
all_data["Fence"] = all_data["Fence"].fillna("None" )
del [data_train]
gc.collect()<init_hyperparams>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
parameters = {
'eavl_metric' : 'logloss',
'max_depth' : 5,
'colsample_bytree' : 0.4,
'subsample' : 0.8
}<train_model>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
xgb.fit(data_train.drop(['reordered', 'uxp_ratio_last_five'], axis=1), data_train.reordered )<predict_on_test>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
y_pred_test =(xgb.predict_proba(data_test.drop('uxp_ratio_last_five', axis=1)) [:, 1] >= 0.21 ).astype('int' )<prepare_output>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
data_test['prediction'] = y_pred_test
data_test.head()<load_from_csv>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
orders = pd.read_csv(PATH + 'orders.csv')
 orders_test = orders.loc[orders.eval_set == 'test', ['user_id', 'order_id']]
orders_test.head()<merge>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
final = final.drop('user_id', axis=1 )<data_type_conversions>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
all_data = all_data.drop(['Utilities'], axis=1 )
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect() 
<prepare_output>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()<save_to_csv>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
sub.to_csv('sub.csv', index=False, header=True )<drop_column>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
del [X, y, X_train, y_train, y_test, X_test, xgb, y_pred]
gc.collect()<set_options>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
gc.enable()<define_variables>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
PATH = '/kaggle/input/instacart-market-basket-analysis/'<load_from_csv>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )
print('loading files...')
 order_products_prior = pd.read_csv(PATH + 'order_products__prior.csv')
 order_products_train = pd.read_csv(PATH + 'order_products__train.csv')
 orders = pd.read_csv(PATH + 'orders.csv')
 products = pd.read_csv(PATH + 'products.csv', usecols=['product_id', 'aisle_id', 'department_id'])
 orders.eval_set = orders.eval_set.replace({'prior': 0, 'train': 1, 'test':2})
 orders.days_since_prior_order = orders.days_since_prior_order.fillna(30)
 print('done loading' )<merge>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
prior_orders = pd.merge(orders, order_products_prior, on='order_id', how='inner')
 prior_orders.head()<drop_column>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
del order_products_prior
gc.collect()<data_type_conversions>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
users = prior_orders.groupby(by='user_id')['order_number'].aggregate('max' ).to_frame('u_num_of_orders' ).reset_index()
users.head()<create_dataframe>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
total_prd_per_order = prior_orders.groupby(by=['user_id', 'order_id'])['product_id'].aggregate('count' ).to_frame('total_products_per_order' ).reset_index()
avg_products = total_prd_per_order.groupby(by=['user_id'])['total_products_per_order'].mean().to_frame('u_avg_prd' ).reset_index()
avg_products.head()
del [total_prd_per_order]
gc.collect()
avg_products.head() 
<data_type_conversions>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
reorder_u = prior_orders.groupby(by='user_id')['reordered'].aggregate('mean' ).to_frame('u_reorder_ratio' ).reset_index()
reorder_u['u_reorder_ratio'] = reorder_u['u_reorder_ratio'].astype(np.float16)
 reorder_u.head()<prepare_output>
all_data = pd.get_dummies(all_data)
 print(all_data.shape)
avg_days = prior_orders.groupby(by='user_id')['days_since_prior_order'].aggregate('mean' ).to_frame('average_days_between_orders')
 avg_days = avg_days.reset_index()
avg_days.head()<data_type_conversions>
train = all_data[:ntrain]
test = all_data[ntrain:]
total_item = prior_orders.groupby(by='user_id' ).size().to_frame('u_total_items_bought' ).astype(np.int16)
 total_item.head()<merge>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
users = users.merge(avg_products, on='user_id', how='left')
 users = users.merge(dow, on='user_id', how='left')
 users = users.merge(hod, on='user_id', how='left')
 users = users.merge(reorder_u, on='user_id', how='left')
 users = users.merge(avg_days, on='user_id', how='left')
 users = users.merge(total_item, on='user_id', how='left')
 users.head()<drop_column>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
del [reorder_u, dow, hod, avg_products, avg_days, total_item]
gc.collect()<data_type_conversions>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
prd = prior_orders.groupby(by='product_id')['order_id'].aggregate('count' ).to_frame('p_num_of_times' ).reset_index()
prd.head()<data_type_conversions>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
reorder_p = prior_orders.groupby(by='product_id')['reordered'].aggregate('mean' ).to_frame('p_reorder_ratio' ).reset_index()
reorder_p.head()<data_type_conversions>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
add_to_cart = prior_orders.groupby(by='product_id')['add_to_cart_order'].aggregate('mean' ).to_frame('p_avg_cart_position' ).reset_index()
add_to_cart.head()<merge>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.03,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
prd = prd.merge(reorder_p, on='product_id', how='left')
 prd = prd.merge(add_to_cart, on='product_id', how='left')
 del [reorder_p, add_to_cart]
gc.collect()<data_type_conversions>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
uxp = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('uxp_times_bought')
 uxp = uxp.reset_index()
uxp.head()<data_type_conversions>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
times = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('times_bought')
 times = times.reset_index()
times.head()<prepare_output>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
total_orders = prior_orders.groupby('user_id')['order_number'].max().to_frame('total_orders' ).reset_index()
total_orders.head()<merge>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
span = pd.merge(total_orders, first_order_num, on='user_id', how='right')
 span.head()<feature_engineering>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
span['Order_Range_D'] = span.total_orders - span.first_order_num + 1
span.head()<merge>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.times_bought / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
uxp_ratio.drop(['times_bought', 'total_orders', 'first_order_num', 'Order_Range_D'], axis=1, inplace=True)
 uxp_ratio.head()<drop_column>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
del [times, span, first_order_num, total_orders]
gc.collect()<merge>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
gc.collect()<data_type_conversions>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
<groupby>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
prior_orders['order_number_back'] = prior_orders.groupby(by=['user_id'])['order_number'].transform(max)- prior_orders.order_number + 1
prior_orders.head()<data_type_conversions>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
last_five['uxp_ratio_last_five'] = last_five.uxp_last_five / 5.0
last_five.head()<data_type_conversions>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
<merge>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
uxp = uxp.merge(last_five, on=['user_id', 'product_id'], how='left')
 del [last_five, temp]
gc.collect()
uxp.head()<correct_missing_values>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
uxp.fillna(0, inplace=True)
 uxp.head(10 )<merge>

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
data = uxp.merge(users, on='user_id', how='left')
 data.head()<merge>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
data = data.merge(prd, on='product_id', how='left')
 data.head()<drop_column>
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
<filter><EOS> 
sub.to_csv('submission.csv',index=False )