Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<categorify>
%matplotlib inline
plt.style.use('seaborn-darkgrid' )
data = pd.get_dummies(data)
 print(data.shape )<prepare_x_and_y>
df_cal = pd.read_csv('.. /input/m5-forecasting-accuracy/calendar.csv')
 df_eval = pd.read_csv('.. /input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 df_price = pd.read_csv('.. /input/m5-forecasting-accuracy/sell_prices.csv')
 df_sample_output = pd.read_csv('.. /input/m5-forecasting-accuracy/sample_submission.csv' )
train = data[:1460]
test = data[1460:].drop('SalePrice', axis = 1 )<prepare_x_and_y>
holiday = ['NewYear', 'OrthodoxChristmas', 'MartinLutherKingDay', 'SuperBowl', 'PresidentsDay', 'StPatricksDay', 'Easter', 'Cinco De Mayo', 'IndependenceDay', 'EidAlAdha', 'Thanksgiving', 'Christmas']
weekend = ['Saturday', 'Sunday']
def is_holiday(x):
if x in holiday:
return 1
else:
return 0
def is_weekend(x):
if x in weekend:
return 1
else:
return 0
X = train.drop('SalePrice', axis = 1)
 y = train['SalePrice']<split>
df_cal['is_holiday_1'] = df_cal['event_name_1'].apply(is_holiday)
 df_cal['is_holiday_2'] = df_cal['event_name_2'].apply(is_holiday)
 df_cal['is_holiday'] = df_cal[['is_holiday_1','is_holiday_2']].max(axis=1)
 df_cal['is_weekend'] = df_cal['weekday'].apply(is_weekend )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 42 )<choose_model_class>
df_cal = df_cal.drop(['weekday', 'wday', 'month', 'year', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'], axis='columns' )
xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1,colsample_bytree=0.6,
gamma=0, importance_type='gain',
learning_rate=0.01, max_delta_step=0,
max_depth=4,min_child_weight=1.5,
n_estimators=2500, n_jobs=1, nthread=None,
objective='reg:linear', reg_alpha=0.4640,
reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1 )<choose_model_class>
del_col = []
for x in range(1851):
del_col.append('d_' + str(x+1))
lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=11000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4 )<choose_model_class>
df_eval = df_eval.drop(del_col, axis='columns' )
gboost = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state =5 )<train_model>
df_eval = pd.merge(df_eval, df_cal, how='left', on='d')
 df_eval.head()
xgb.fit(X_train, y_train)
 lgbm.fit(X_train, y_train, eval_metric='rmsle')
 gboost.fit(X_train, y_train )<predict_on_test>
df_eval = pd.merge(df_eval, df_price, how='left', on=['item_id', 'wm_yr_wk', 'store_id'])
 df_eval.head()
pred2 = xgb.predict(X_test)
 pred3 = lgbm.predict(X_test)
 pred4 = gboost.predict(X_test )<compute_test_metric>
df_eval_test = df_eval.query('d == "d_1852"' )
print('Root Mean Square Logarithmic Error test(XGB)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred2))))
print('Root Mean Square Logarithmic Error test(LGBM)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred3))))
print('Root Mean Square Logarithmic Error test(GBoost)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred4))))<train_model>
df_eval_test = df_eval_test[['id', 'store_id', 'item_id', 'dept_id', 'cat_id', 'state_id', 'd', 'qty', 'sell_price']]
lgbm.fit(X, y)
 xgb.fit(X ,y)
 gboost.fit(X, y )<predict_on_test>
df_eval_test['qty'] = df_eval_test['d'].apply(lambda x: int(x.replace(x, '0')) )
prediction_lgbm = np.expm1(lgbm.predict(test))
prediction_xgb = np.expm1(xgb.predict(test))
prediction_gboost = np.expm1(gboost.predict(test))<define_variables>
tmp_df = df_eval_test
<prepare_output>
for x in range(28):
df_eval_test = df_eval_test.append(tmp_df )
prediction =(prediction_lgbm * 0.38 + prediction_gboost * 0.35 + prediction_xgb * 0.27 )<create_dataframe>
df_eval_test = df_eval_test.reset_index(drop=True )
submission = pd.DataFrame({"Id": test.index,"SalePrice": prediction} )<save_to_csv>
lst_d = []
i = 0
lst_index = df_eval_test.index
for x in lst_index:
lst_d.append('d_' + str(((lst_index[i])// 30490)+ 1942))
i = i + 1
lst_d
submission.to_csv('submission.csv', index=False )<save_to_csv>
df_eval_test['d'] = lst_d
submission.to_csv('submission.csv', index=False )<import_modules>
df_eval_test = pd.merge(df_eval_test, df_cal, how='left', on='d' )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
%matplotlib inline<load_from_csv>
df_eval_test = pd.merge(df_eval_test, df_price, how='left', on=['item_id', 'wm_yr_wk', 'store_id'] )
train_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 sample_submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 train_data.head(5 )<prepare_x_and_y>
del tmp_df
gc.collect()
y_train = train_data['SalePrice']
X_train = train_data.drop('SalePrice',axis=1)
 dataset = pd.concat(objs = [X_train,test_data],axis=0,sort=False ).reset_index(drop=True)
 print(len(dataset))
dataset.head(5 )<count_missing_values>
df_eval = pd.get_dummies(data=df_eval, columns=['dept_id', 'cat_id', 'store_id', 'state_id'])
 df_eval_test = pd.get_dummies(data=df_eval_test, columns=['dept_id', 'cat_id', 'store_id', 'state_id'] )
dataset_null_val = dataset.isnull().sum()
print('For Dataset
')
 print('{} no of features in dataset contain missing values
'.format(len(dataset_null_val.values[dataset_null_val.values !=0])))
 print('column names with null values {}
'.format(dataset.columns[dataset_null_val.values !=0])) 
<create_dataframe>
df_eval_test = df_eval_test.drop(['sell_price_x', 'snap_CA', 'snap_TX', 'snap_WI'], axis='columns')
 df_eval_test = df_eval_test.rename(columns={'sell_price_y': 'sell_price'})
 df_eval = df_eval.drop(['snap_CA', 'snap_TX', 'snap_WI'], axis='columns' )
NAN = [(c, dataset[c].isna().mean() *100)for c in dataset]
NAN = pd.DataFrame(NAN, columns=["column_name", "percentage"])
 NAN<sort_values>
target_col = 'qty'
exclude_cols = ['id', 'item_id', 'd', 'date', 'wm_yr_wk']
feature_cols = [col for col in df_eval.columns if col not in exclude_cols]
y = np.array(df_eval[target_col])
 X = np.array(df_eval[feature_cols])
 X_train, X_test, y_train, y_test = \
train_test_split(X, y, test_size=0.3, random_state=1234)
NAN = NAN[NAN.percentage > 50]
NAN.sort_values("percentage", ascending=False )<drop_column>
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_test, y_test)
 params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'learning_rate': 0.01,
'bagging_fraction': 0.75,
'bagging_freq': 10,
'colsample_bytree': 0.75}
model = lgb.train(params, lgb_train, num_boost_round=2500, early_stopping_rounds=50, valid_sets = [lgb_train, lgb_eval], verbose_eval=100 )
dataset = dataset.drop(['Alley','PoolQC','Fence','MiscFeature'],axis=1)
 dataset.shape[1]<define_variables>
pred = model.predict(df_eval_test[feature_cols] )
object_columns_df = dataset.select_dtypes(include=['object'])
 numerical_columns_df =dataset.select_dtypes(exclude=['object'])
<count_missing_values>
df_eval_test['pred_qty'] = pred
null_counts = object_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<data_type_conversions>
predictions = df_eval_test[['id', 'date', 'pred_qty']]
predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'pred_qty' ).reset_index()
predictions
columns_None = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','FireplaceQu','GarageCond']
object_columns_df[columns_None]= object_columns_df[columns_None].fillna('None' )<categorify>
predictions = predictions.drop(predictions.columns[1], axis=1)
 predictions
columns_with_lowNA = ['MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','Electrical','KitchenQual','Functional','SaleType']
object_columns_df[columns_with_lowNA] = object_columns_df[columns_with_lowNA].fillna(object_columns_df.mode().iloc[0] )<count_missing_values>
predictions.columns = ['id'] + ['F' + str(i + 1)for i in range(28)]
predictions
null_counts = numerical_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<feature_engineering>
x = 2744099 + 1 - 853720
df_val = df_eval[x:]
numerical_columns_df['GarageYrBlt'] = numerical_columns_df['GarageYrBlt'].fillna(numerical_columns_df['YrSold']-35)
 numerical_columns_df['LotFrontage'] = numerical_columns_df['LotFrontage'].fillna(68)
<drop_column>
predictions_v = df_val[['id', 'date', 'qty']]
predictions_v = pd.pivot(predictions_v, index = 'id', columns = 'date', values = 'qty' ).reset_index()
predictions_v
object_columns_df = object_columns_df.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1 )<filter>
predictions_v['id'] = predictions['id'].apply(lambda x: x.replace('evaluation', 'validation'))
predictions_v.head()
Negatif = numerical_columns_df[numerical_columns_df['Age_House'] < 0]
Negatif<categorify>
predictions_v.columns = ['id'] + ['F' + str(i + 1)for i in range(28)]
predictions_v.head()
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,"None" : 0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6
}
object_columns_df['ExterQual'] = object_columns_df['ExterQual'].map(bin_map)
 object_columns_df['ExterCond'] = object_columns_df['ExterCond'].map(bin_map)
 object_columns_df['BsmtCond'] = object_columns_df['BsmtCond'].map(bin_map)
 object_columns_df['BsmtQual'] = object_columns_df['BsmtQual'].map(bin_map)
 object_columns_df['HeatingQC'] = object_columns_df['HeatingQC'].map(bin_map)
 object_columns_df['KitchenQual'] = object_columns_df['KitchenQual'].map(bin_map)
 object_columns_df['FireplaceQu'] = object_columns_df['FireplaceQu'].map(bin_map)
 object_columns_df['GarageQual'] = object_columns_df['GarageQual'].map(bin_map)
 object_columns_df['GarageCond'] = object_columns_df['GarageCond'].map(bin_map)
 object_columns_df['CentralAir'] = object_columns_df['CentralAir'].map(bin_map)
 object_columns_df['LotShape'] = object_columns_df['LotShape'].map(bin_map)
 object_columns_df['BsmtExposure'] = object_columns_df['BsmtExposure'].map(bin_map)
 object_columns_df['BsmtFinType1'] = object_columns_df['BsmtFinType1'].map(bin_map)
 object_columns_df['BsmtFinType2'] = object_columns_df['BsmtFinType2'].map(bin_map)
 PavedDrive = {"N" : 0, "P" : 1, "Y" : 2}
object_columns_df['PavedDrive'] = object_columns_df['PavedDrive'].map(PavedDrive )<categorify>
predictions_concat = pd.concat([predictions, predictions_v], axis=0 )
rest_object_columns = object_columns_df.select_dtypes(include=['object'])
 object_columns_df = pd.get_dummies(object_columns_df, columns=rest_object_columns.columns )<normalization>
predictions_concat.to_csv('submission.csv', index=False )