Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<remove_duplicates>
init_notebook_mode(connected=True)
 cf.go_offline()
%matplotlib inline
relation = data[['user_id','product_id']].copy()
relation.drop_duplicates(inplace=True )<prepare_output>
data_desc = '.. /input/house-prices-advanced-regression-techniques/data_description.txt'
train_path = '.. /input/house-prices-advanced-regression-techniques/train.csv'
test_path = '.. /input/house-prices-advanced-regression-techniques/test.csv'
sub = orders[orders.eval_set == 'test']
sub = sub[['order_id','user_id']]<create_dataframe>
data_desc = open(data_desc, mode='r')
 for line in data_desc.readlines() :
print(line[:-1] )
train_data = turicreate.SFrame(data)
 test_data = turicreate.SFrame(test)
 relation_data = turicreate.SFrame(relation )<drop_column>
train_data = pd.read_csv(train_path, index_col=0)
 test_data = pd.read_csv(test_path, index_col=0 )
def clean_prediction(row):
data = row.products
data = str("".join(str(data)) [1:-1].replace(',',' '))
return data<prepare_output>
print('Missing Values in Train Data:
')
 print(train_data.isna().sum() [train_data.isna().sum() != 0] )
result = recommender.recommend(users=test_data, items=relation_data, exclude_known=False, k=len(products)).to_dataframe()<feature_engineering>
print('Missing Values in Test Data :
')
 print(test_data.isna().sum() [test_data.isna().sum() != 0] )
for key,item in thresholds.items() :
print(key,':',str(item))
results[key] = result[result['score'] > item]
results[key] = results[key].groupby('user_id')['product_id'].apply(list ).reset_index(name='products')
 results[key]['products'] = results[key].apply(clean_prediction, axis=1 )<merge>
y = train_data['SalePrice']
data = pd.concat([train_data.drop('SalePrice', axis=1), test_data])
 data.head(3 )
for key,item in thresholds.items() :
results[key] = pd.merge(sub, results[key],how='outer',on='user_id' ).sort_values('user_id')
 results[key].fillna('None', inplace=True)
 results[key].drop('user_id',axis=1,inplace=True )<save_to_csv>
def missing_data_func(feature_type):
if feature_type.lower() [0] == 'c':
data_type = ['object']
elif feature_type.lower() [0] == 'n':
data_type = ['integer', 'float']
else :
data_type = ['integer', 'float', 'object']
missing_value = data.select_dtypes(include=data_type ).isna().sum() [data.select_dtypes(include=data_type ).isna().sum() != 0].sort_values(ascending=False)
 missing_percentage =(missing_value/2919)*100
missing_data = pd.DataFrame(data=[missing_value, missing_percentage], index=['Count', 'Percentage'])
 return missing_data
def visualize_missing_data(Data, title):
sns.set(font_scale=1.3)
 plt.figure(figsize=(20, 5))
sns.barplot(x=Data.T.index, y=Data.T['Percentage'], palette="rocket")
 plt.xticks(rotation=90)
 plt.title(label=title, fontdict={'fontsize':18})
 plt.show()
for key,item in thresholds.items() :
results[key].to_csv('submission_turi_ItemSim_pearson_pol'+str(key)+'.csv',index=False )<set_options>
data.drop(labels=['YearRemodAdd', 'LotArea', '1stFlrSF', 'TotRmsAbvGrd', 'GarageYrBlt', 'GarageCars'], axis=1, inplace=True)
 data.drop(labels=['MSSubClass', 'OverallCond', 'BsmtFinSF2', 'BsmtUnfSF', 'BsmtFullBath', 'LowQualFinSF', 'BsmtHalfBath',
'BedroomAbvGr', 'KitchenAbvGr', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'MoSold', 'YrSold'], axis=1, inplace=True )
gc.enable()<define_variables>
data.drop(labels=['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu'], axis=1, inplace=True )
PATH = '/kaggle/input/instacart-market-basket-analysis/'<load_from_csv>
categorical = dict()
for column in data.select_dtypes(include=['object'] ).columns:
categorical[column] =(train_data[column].value_counts() [0].sum() , 
(train_data[column].value_counts() [0].sum() /train_data[column].value_counts().sum())*100,
train_data[column].value_counts().shape[0])
 dominant_features = pd.DataFrame(data=categorical.values() , index=categorical.keys() , columns=['Count', 'Percentage',
'variable type'] ).sort_values(by='Percentage')
 dominant_features
print('loading files...')
 order_products_prior = pd.read_csv(PATH + 'order_products__prior.csv')
 order_products_train = pd.read_csv(PATH + 'order_products__train.csv')
 orders = pd.read_csv(PATH + 'orders.csv')
 products = pd.read_csv(PATH + 'products.csv', usecols=['product_id', 'aisle_id', 'department_id'])
 orders.eval_set = orders.eval_set.replace({'prior': 0, 'train': 1, 'test':2})
 orders.days_since_prior_order = orders.days_since_prior_order.fillna(30)
 print('done loading' )<merge>
data.drop(labels=dominant_features[dominant_features['Percentage'] > 80].index,axis=1, inplace=True )
prior_orders = pd.merge(orders, order_products_prior, on='order_id', how='inner')
 prior_orders.head()<drop_column>
num_data = data.select_dtypes(include=['integer', 'float'])
 for features in num_data.isna().sum() [num_data.isna().sum() != 0].index:
data[features].fillna(data[features].mean() , inplace=True)
 cat_data = data.select_dtypes(include=['object'])
 for features in cat_data.isna().sum() [cat_data.isna().sum() != 0].index:
data[features].fillna(data[features].mode() [0], inplace=True)
 data.isna().sum() [data.isna().sum() != 0]
del order_products_prior
gc.collect()<data_type_conversions>
ms_zoning = {'C(all)':0, 'RM':1, 'RH':1, 'RL':2, 'FV':3}
lot_shape = {'Reg':0, 'IR1':1, 'IR2':2, 'IR3':2}
lot_config = {'Inside':0, 'FR2':1, 'Corner':2, 'CulDSac':3, 'FR3':3}
neighbor_hood = {'MeadowV':0, 'IDOTRR':0, 'BrDale':0, 'BrkSide':1, 'Edwards':1, 'OldTown':1,
'Sawyer':2, 'Blueste':2, 'SWISU':2, 'NPkVill':2, 'NAmes':3, 'Mitchel':3, 'SawyerW':4,
'NWAmes':5, 'Gilbert':5, 'Blmngtn':5, 'CollgCr':6, 'Crawfor':6, 'ClearCr':7,
'Somerst':7, 'Veenker':8, 'Timber':8, 'StoneBr':9, 'NridgHt':9, 'NoRidge':10}
house_style = {'1.5Unf':0, 'SFoyer':1, '1.5Fin':1, '2.5Unf':1, 'SLvl':1, '1Story':2, '2Story':3, '2.5Fin':3}
roof_style = {'Gambrel':0, 'Gable':0, 'Mansard':1, 'Flat':1, 'Hip':1, 'Shed':1}
exterior_1st = {'BrkComm':0, 'AsphShn':0, 'CBlock':0, 'AsbShng':0, 'MetalSd':1, 'Wd Sdng':1,
'WdShing':2, 'Stucco':3, 'HdBoard':3, 'Plywood':3, 'BrkFace':4, 'VinylSd':5,
'CemntBd':6, 'Stone':6, 'ImStucc':6}
exterior_2nd = {'CBlock':0, 'AsbShng':0, 'Brk Cmn':0, 'AsphShn':0, 'Wd Sdng':1, 'MetalSd':1,
'Stucco':2, 'Stone':2, 'Wd Shng':2, 'HdBoard':3, 'Plywood':3, 'BrkFace':4,
'VinylSd':4, 'CmentBd':5, 'ImStucc':5, 'Other':5}
masvnr_type = {'BrkCmn':0, 'None':0, 'BrkFace':1, 'Stone':2}
exter_qual = {'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
foundation = {'Slab':0, 'BrkTil':1, 'CBlock':2, 'Stone':3, 'Wood':3, 'PConc':3}
bsmt_qual = {'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
bsmt_exposure = {'No':0, 'Mn':1, 'Av':2, 'Gd':3}
bsmtfin_type1 = {'Rec':0, 'BLQ':0, 'LwQ':1, 'ALQ':1, 'Unf':2, 'GLQ':3}
heat_qc = {'Po':0, 'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
kitchen_qual = {'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
garage_type = {'CarPort':0, 'Detchd':0, '2Types':1, 'Basment':1, 'Attchd':2, 'BuiltIn':3}
garage_finish = {'Unf':0, 'RFn':1, 'Fin':2}
users = prior_orders.groupby(by='user_id')['order_number'].aggregate('max' ).to_frame('u_num_of_orders' ).reset_index()
users.head()<create_dataframe>
data['MSZoning'] = data['MSZoning'].map(ms_zoning)
 data['LotShape'] = data['LotShape'].map(lot_shape)
 data['LotConfig'] = data['LotConfig'].map(lot_config)
 data['Neighborhood'] = data['Neighborhood'].map(neighbor_hood)
 data['HouseStyle'] = data['HouseStyle'].map(house_style)
 data['RoofStyle'] = data['RoofStyle'].map(roof_style)
 data['Exterior1st'] = data['Exterior1st'].map(exterior_1st)
 data['Exterior2nd'] = data['Exterior2nd'].map(exterior_2nd)
 data['MasVnrType'] = data['MasVnrType'].map(masvnr_type)
 data['ExterQual'] = data['ExterQual'].map(exter_qual)
 data['Foundation'] = data['Foundation'].map(foundation)
 data['BsmtQual'] = data['BsmtQual'].map(bsmt_qual)
 data['BsmtExposure'] = data['BsmtExposure'].map(bsmt_exposure)
 data['BsmtFinType1'] = data['BsmtFinType1'].map(bsmtfin_type1)
 data['HeatingQC'] = data['HeatingQC'].map(heat_qc)
 data['KitchenQual'] = data['KitchenQual'].map(kitchen_qual)
 data['GarageType'] = data['GarageType'].map(garage_type)
 data['GarageFinish'] = data['GarageFinish'].map(garage_finish )
total_prd_per_order = prior_orders.groupby(by=['user_id', 'order_id'])['product_id'].aggregate('count' ).to_frame('total_products_per_order' ).reset_index()
avg_products = total_prd_per_order.groupby(by=['user_id'])['total_products_per_order'].mean().to_frame('u_avg_prd' ).reset_index()
avg_products.head()
del [total_prd_per_order]
gc.collect()
avg_products.head() 
<data_type_conversions>
outlier_data = data[:][:1460]
outlier_data['SalePrice'] = y
outlier_data.drop(labels=outlier_data[outlier_data['LotFrontage']>300].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['YearBuilt']<1900)&(outlier_data['SalePrice']>300000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['YearBuilt']<2000)&(outlier_data['SalePrice']>600000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[outlier_data['MasVnrArea']>1160].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[outlier_data['SalePrice']>700000].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['SalePrice']>700000)|(outlier_data['BsmtFinSF1']>5000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['TotalBsmtSF']>5000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['GrLivArea']>4000)&(outlier_data['SalePrice']<300000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['GarageArea']>1200)&(outlier_data['SalePrice']<300000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[outlier_data['SalePrice']>700000].index, axis=0, inplace=True )
reorder_u = prior_orders.groupby(by='user_id')['reordered'].aggregate('mean' ).to_frame('u_reorder_ratio' ).reset_index()
reorder_u['u_reorder_ratio'] = reorder_u['u_reorder_ratio'].astype(np.float16)
 reorder_u.head()<prepare_output>
y = outlier_data['SalePrice']
train_data = outlier_data.drop(labels='SalePrice', axis=1)
 test_data = data[1460:]
avg_days = prior_orders.groupby(by='user_id')['days_since_prior_order'].aggregate('mean' ).to_frame('average_days_between_orders')
 avg_days = avg_days.reset_index()
avg_days.head()<data_type_conversions>
scaler = StandardScaler()
y = pd.DataFrame(data=scaler.fit_transform(np.array(y ).reshape(1448,1)) , index=y.index, columns=['SalePrice'])
 y = np.ravel(y )
total_item = prior_orders.groupby(by='user_id' ).size().to_frame('u_total_items_bought' ).astype(np.int16)
 total_item.head()<merge>
def cross_val(model, x):
score = -cross_val_score(estimator=model, X=x, y=y, scoring='neg_root_mean_squared_error', cv=10, verbose=1, n_jobs=2 ).mean()
return score
users = users.merge(avg_products, on='user_id', how='left')
 users = users.merge(dow, on='user_id', how='left')
 users = users.merge(hod, on='user_id', how='left')
 users = users.merge(reorder_u, on='user_id', how='left')
 users = users.merge(avg_days, on='user_id', how='left')
 users = users.merge(total_item, on='user_id', how='left')
 users.head()<drop_column>
linear_reg = make_pipeline(StandardScaler() , LinearRegression())
 poly_feat = PolynomialFeatures(degree=2)
 poly_reg = make_pipeline(poly_feat, StandardScaler() , LinearRegression())
 ridge_reg = make_pipeline(StandardScaler() , RidgeCV(alphas=np.linspace(start=0.00001, stop=100, num=350), scoring='r2', cv=8))
lasso_reg = make_pipeline(StandardScaler() , LassoCV(alphas=np.linspace(start=0.00001, stop=100, num=350), cv=8))
elasticnet_reg = make_pipeline(StandardScaler() , ElasticNetCV(l1_ratio=[.2,.4,.5,.6,.8],
alphas=np.linspace(start=0.00001, stop=100, num=350), cv=8, max_iter=1000))
svr = make_pipeline(StandardScaler() , SVR(C=10, epsilon=0.0001))
decision_tree = DecisionTreeRegressor(ccp_alpha=0.0, criterion='mse', max_depth=15,
max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=10, min_samples_split=2,
min_weight_fraction_leaf=0, presort='deprecated',
random_state=None, splitter='best')
 rfr = RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',
max_depth=40, max_features='auto', max_leaf_nodes=None,
max_samples=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0,
n_estimators=100, n_jobs=None, oob_score=False,
random_state=None, verbose=0, warm_start=False)
 adaboost_reg = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(ccp_alpha=0.0,
criterion='mse', max_depth=None, max_features=None,
max_leaf_nodes=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0.0,
presort='deprecated', random_state=None,
splitter='best'),
learning_rate=1, loss='square', n_estimators=350,
random_state=None)
 gbr = GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',
init=None, learning_rate=0.02, loss='ls', max_depth=7,
max_features='sqrt', max_leaf_nodes=25,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=550,
n_iter_no_change=None, presort='deprecated',
random_state=None, subsample=0.65, tol=0.0001,
validation_fraction=0.1, verbose=0, warm_start=False)
 lgbr = LGBMRegressor(boosting='gbdt', boosting_type='gbdt', class_weight=None,
colsample_bytree=1.0, importance_type='split', subsample_freq=0,
learning_rate=0.02, max_depth=7, min_child_samples=20,
min_child_weight=0.001, min_split_gain=0.0, n_estimators=450,
n_jobs=-1, num_leaves=18, objective=None, random_state=None,
reg_alpha=0.0, reg_lambda=0.0, silent=True, subsample=1.0,
subsample_for_bin=200000)
 xgbr = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0.1,
importance_type='gain', learning_rate=0.02, max_delta_step=0,
max_depth=3, min_child_weight=1, missing=None, n_estimators=750,
n_jobs=-1, nthread=None, objective='reg:linear', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=0.65, verbosity=1 )
del [reorder_u, dow, hod, avg_products, avg_days, total_item]
gc.collect()<data_type_conversions>
estimator_list = {'Linear Regressor': linear_reg,
'Polynomial Regressor': poly_reg,
'Ridge Regressor': ridge_reg,
'Lasso Regressor': lasso_reg,
'ElasticNet Regressor': elasticnet_reg,
'Support Vector Regressor': svr,
'Decision Tree': decision_tree,
'Random Forest Regressor': rfr,
'Adaboost Regressor': adaboost_reg,
'Gradient Boosting Regressor': gbr,
'Light Gradient Boosting Regressor': lgbr,
'Extreme Gradient Boosting Regressor': xgbr}
score = {}
def results() :
for model in estimator_list.keys() :
score[model] = cross_val(model= estimator_list[model], x= train_data)
 results()
prd = prior_orders.groupby(by='product_id')['order_id'].aggregate('count' ).to_frame('p_num_of_times' ).reset_index()
prd.head()<data_type_conversions>
for name in estimator_list.keys() :
print(name.upper() , ':')
 estimator_list[name].fit(train_data, y)
 print('\t\tDONE !
' )
reorder_p = prior_orders.groupby(by='product_id')['reordered'].aggregate('mean' ).to_frame('p_reorder_ratio' ).reset_index()
reorder_p.head()<data_type_conversions>
stacked_reg = StackingRegressor(estimators=[('Linear Regressor', linear_reg), 
('Polynomial Regressor', poly_reg), 
('Ridge Regressor', ridge_reg), 
('Lasso Regressor', lasso_reg), 
('ElasticNet Regressor', elasticnet_reg), 
('Support Vector Regressor', svr), 
('Decision Tree', decision_tree), 
('Random Forest Regressor', rfr), 
('Adaboost Regressor', adaboost_reg), 
('Gradient Boosting Regressor', gbr), 
('Light Gradient Boosting Regressor', lgbr), 
('Extreme Gradient Boosting Regressor', xgbr)],
cv=7, n_jobs=-1, verbose=1 )
add_to_cart = prior_orders.groupby(by='product_id')['add_to_cart_order'].aggregate('mean' ).to_frame('p_avg_cart_position' ).reset_index()
add_to_cart.head()<merge>
params={'final_estimator':[estimator_list[name] for name in estimator_list.keys() ]}
search = GridSearchCV(estimator=stacked_reg, param_grid=params,
scoring='neg_root_mean_squared_error', cv=7, verbose=1, n_jobs=-1)
 search.fit(train_data, y )
prd = prd.merge(reorder_p, on='product_id', how='left')
 prd = prd.merge(add_to_cart, on='product_id', how='left')
 del [reorder_p, add_to_cart]
gc.collect()<data_type_conversions>
search.best_estimator_
uxp = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('uxp_times_bought')
 uxp = uxp.reset_index()
uxp.head()<data_type_conversions>
search.best_score_
times = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('times_bought')
 times = times.reset_index()
times.head()<prepare_output>
stacked_reg = search.best_estimator_
stacked_reg.fit(X=train_data, y=y )
total_orders = prior_orders.groupby('user_id')['order_number'].max().to_frame('total_orders' ).reset_index()
total_orders.head()<merge>
final_pred = scaler.inverse_transform(stacked_reg.predict(test_data))
span = pd.merge(total_orders, first_order_num, on='user_id', how='right')
 span.head()<feature_engineering>
submission = pd.DataFrame()
submission['Id'] = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')['Id']
submission['SalePrice'] = final_pred
submission.to_csv('House Price Prediction.csv',index=False)
 pd.read_csv('House Price Prediction.csv', index_col=0 ).head(8 )