Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<merge>
%matplotlib inline
data = data.merge(orders_future, on='user_id', how='left')
 data.head()<split>
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
data_train = data[data.eval_set == 1]
data_train.head()<merge>
train.drop('Id', axis= 1, inplace= False)
data_train = data_train.merge(order_products_train[['product_id', 'order_id', 'reordered']], on=['product_id', 'order_id'], how='left')
 data_train.head()<drop_column>
test.drop('Id', axis= 1, inplace= False )
data_train.drop(['eval_set', 'order_id'], axis=1, inplace=True )<split>
train1= train
test1= test
data_test = data[data.eval_set == 2]
data_test.head()<drop_column>
df = pd.concat([train1, test1], axis=0,sort=False)
data_test.drop(['eval_set', 'order_id'], axis=1, inplace=True )<merge>
df.isnull().sum()
products = pd.read_csv(PATH + 'products.csv', usecols=['product_id', 'aisle_id', 'department_id'])
 data_train = data_train.merge(products, on='product_id', how='left')
 data_test = data_test.merge(products, on='product_id', how='left')
 data_train = data_train.set_index(['user_id', 'product_id'])
 data_test = data_test.set_index(['user_id', 'product_id'] )<categorify>
df["PoolQC"] = df["PoolQC"].fillna("None")
 df["MiscFeature"] = df["MiscFeature"].fillna("None")
 df["Alley"] = df["Alley"].fillna("None")
 df['Fence'] = df['Fence'].fillna('None')
 df['PoolQC'] = df['PoolQC'].fillna('None')
 df['GarageCond'] = df['GarageCond'].fillna('None')
 df['GarageQual'] = df['GarageQual'].fillna('None')
 df['GarageFinish'] = df['GarageFinish'].fillna('None')
 df['GarageType'] = df['GarageType'].fillna('None')
 df['FireplaceQu'] = df['FireplaceQu'].fillna('None')
 df['BsmtFinType2'] = df['BsmtFinType2'].fillna('None')
 df['BsmtFinType1'] = df['BsmtFinType1'].fillna('None')
 df['BsmtExposure'] = df['BsmtExposure'].fillna('None')
 df['BsmtCond'] = df['BsmtCond'].fillna('None')
 df['BsmtQual'] = df['BsmtQual'].fillna('None' )
columns_mean = ['aisle_id', 'department_id']
for col in columns_mean:
mean = data_train.groupby(col ).reordered.mean()
data_train[col] = data_train[col].map(mean)
 data_test[col] = data_test[col].map(mean )<drop_column>
df.isnull().sum()
del [data, orders_future, products, order_products_train]
gc.collect()<import_modules>
df_missing =(df.isnull().sum() / len(df)) * 100
df_missing = df_missing.drop(df_missing[df_missing == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :df_missing})
 missing_data.head(30 )
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.metrics import f1_score, classification_report
from scikitplot.metrics import plot_confusion_matrix
from scikitplot.classifiers import plot_feature_importances
import xgboost as xgb
import lightgbm as lgb<split>
df = df.fillna(df.median())
 df_missing =(df.isnull().sum() / len(df)) * 100
df_missing = df_missing.drop(df_missing[df_missing == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :df_missing})
 missing_data.head(30 )
X = data_train.drop(['reordered', 'uxp_ratio_last_five'], axis=1)
 y = data_train.reordered
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=10 )<drop_column>
df = df.fillna(df.median() )
del [X, y]
gc.collect()<train_model>
df = df.fillna(df.median())
 df_missing =(df.isnull().sum() / len(df)) * 100
df_missing = df_missing.drop(df_missing[df_missing == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :df_missing})
 missing_data.head(30 )
lgbm.fit(data_train.drop(['reordered', 'uxp_ratio_last_five'], axis=1), data_train.reordered )<predict_on_test>
df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
y_pred_test =(lgbm.predict_proba(data_test.drop('uxp_ratio_last_five', axis=1)) [:, 1] >= 0.21 ).astype('int' )<prepare_output>
df = df.fillna(df.median())
 df_missing =(df.isnull().sum() / len(df)) * 100
df_missing = df_missing.drop(df_missing[df_missing == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :df_missing})
 missing_data.head(30 )
data_test['prediction'] = y_pred_test
data_test.head()<load_from_csv>
df= pd.get_dummies(df, columns= ['MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities',
'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual',
'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature',
'SaleType', 'SaleCondition'] )
orders = pd.read_csv(PATH + 'orders.csv')
 orders_test = orders.loc[orders.eval_set == 'test', ['user_id', 'order_id']]
orders_test.head()<merge>
object_columns_df = object_columns_df.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1)
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
Negatif = numerical_columns_df[numerical_columns_df['Age_House'] < 0]
Negatif
final = final.drop('user_id', axis=1 )<data_type_conversions>
numerical_columns_df['TotalBsmtBath'] = numerical_columns_df['BsmtFullBath'] + numerical_columns_df['BsmtFullBath']*0.5
numerical_columns_df['TotalBath'] = numerical_columns_df['FullBath'] + numerical_columns_df['HalfBath']*0.5
numerical_columns_df['TotalSA']=numerical_columns_df['TotalBsmtSF'] + numerical_columns_df['1stFlrSF'] + numerical_columns_df['2ndFlrSF']
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,"None" : 0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6
}
object_columns_df['ExterQual'] = object_columns_df['ExterQual'].map(bin_map)
 object_columns_df['ExterCond'] = object_columns_df['ExterCond'].map(bin_map)
 object_columns_df['BsmtCond'] = object_columns_df['BsmtCond'].map(bin_map)
 object_columns_df['BsmtQual'] = object_columns_df['BsmtQual'].map(bin_map)
 object_columns_df['HeatingQC'] = object_columns_df['HeatingQC'].map(bin_map)
 object_columns_df['KitchenQual'] = object_columns_df['KitchenQual'].map(bin_map)
 object_columns_df['FireplaceQu'] = object_columns_df['FireplaceQu'].map(bin_map)
 object_columns_df['GarageQual'] = object_columns_df['GarageQual'].map(bin_map)
 object_columns_df['GarageCond'] = object_columns_df['GarageCond'].map(bin_map)
 object_columns_df['CentralAir'] = object_columns_df['CentralAir'].map(bin_map)
 object_columns_df['LotShape'] = object_columns_df['LotShape'].map(bin_map)
 object_columns_df['BsmtExposure'] = object_columns_df['BsmtExposure'].map(bin_map)
 object_columns_df['BsmtFinType1'] = object_columns_df['BsmtFinType1'].map(bin_map)
 object_columns_df['BsmtFinType2'] = object_columns_df['BsmtFinType2'].map(bin_map)
 PavedDrive = {"N" : 0, "P" : 1, "Y" : 2}
object_columns_df['PavedDrive'] = object_columns_df['PavedDrive'].map(PavedDrive)
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect() 
<prepare_output>
rest_object_columns = object_columns_df.select_dtypes(include=['object'])
 object_columns_df = pd.get_dummies(object_columns_df, columns=rest_object_columns.columns)
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()<save_to_csv>
df_final = pd.concat([object_columns_df, numerical_columns_df], axis=1,sort=False)
 df_final.head()
sub.to_csv('sub.csv', index=False, header=True )<drop_column>
target= df_train['SalePrice']
df_train = df_train.drop(['SalePrice'],axis=1 )
del [X, y, X_train, y_train, y_test, X_test, lgbm, y_pred]
gc.collect()<set_options>
df_test = df_test.drop(['SalePrice'],axis=1)
 df_test.head()
gc.enable()<load_from_csv>
x_train,x_test,y_train,y_test = train_test_split(df_train,target,test_size=0.2,random_state=0 )
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )<data_type_conversions>
xgb =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=12000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4, 
)
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )<merge>
xgb.fit(x_train, y_train)
 lgbm.fit(x_train, y_train,eval_metric='rmse')
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()<create_dataframe>
predict1 = xgb.predict(x_test)
 predict = lgbm.predict(x_test)
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user.head() 
<prepare_output>
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict1))))
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict))))
dow=op.groupby('user_id')['order_dow'].mean().to_frame('average_dow')
 dow=dow.reset_index()
dow.head()<merge>
xgb.fit(df_train, target)
 lgbm.fit(df_train, target,eval_metric='rmse')
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()<prepare_output>
predict4 = lgbm.predict(df_test)
 predict3 = xgb.predict(df_test)
 predict_y =(predict3*0.575 + predict4 * 0.425)
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd.head()<groupby>
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": predict_y
})
 submission.to_csv('submission.csv', index=False)
 print('Submission completed')