Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<define_variables>
sales_train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 test_ids = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 sales_train = sales_train.drop(['date'], axis=1)
 sales_train.head()
a = df['DAYS_EMPLOYED_PERC'].tolist()
a = [x for x in a if str(x)!= 'nan']
b = df['INCOME_CREDIT_PERC'].tolist()
b = [x for x in b if str(x)!= 'nan']
c = df['INCOME_PER_PERSON'].tolist()
c = [x for x in c if str(x)!= 'nan']
d = df['ANNUITY_INCOME_PERC'].tolist()
d = [x for x in d if str(x)!= 'nan']<train_model>
ids = list(np.arange(test_ids['ID'].max() +1)) *(sales_train['date_block_num'].max() +1)
 dates = list(np.arange(sales_train['date_block_num'].max() +1)) *(test_ids['ID'].max() +1)
 dates.sort()
date_id_dict = {'ID' : ids, 'date_block_num' : dates}
date_id_df = pd.DataFrame.from_dict(date_id_dict)
 date_id_df = date_id_df.merge(test_ids, on='ID')
 date_id_df.head()
print("End Train Test.................. " )<train_model>
grouped = sales_train.groupby(['date_block_num', 'shop_id', 'item_id'], as_index=False)
 item_cnt = pd.DataFrame(grouped.sum())
 item_cnt = item_cnt.drop(['item_price'], axis=1)
 grouped = sales_train.groupby(['shop_id', 'item_id'])
 avg_price = pd.DataFrame(grouped.mean() ['item_price'])
 monthly_sales = item_cnt.merge(avg_price, on=['shop_id', 'item_id'])
 monthly_sales = monthly_sales.merge(test_ids, on=['shop_id', 'item_id'])
 monthly_sales.head()
print("Start Bureau................ " )<load_from_csv>
item_price = monthly_sales[['item_price', 'ID']]
monthly_sales = monthly_sales.drop(['item_price'], axis=1)
 monthly_sales = date_id_df.merge(monthly_sales, how='left', on=['ID', 'date_block_num'])
 monthly_sales = monthly_sales.drop(['shop_id_y', 'item_id_y'], axis=1)
 monthly_sales['item_cnt_day'].fillna(0, inplace=True)
 monthly_sales = monthly_sales.merge(item_price, how='left', on='ID')
 monthly_sales['item_price'].fillna(monthly_sales['item_price'].mean() , inplace=True)
 monthly_sales = monthly_sales.drop_duplicates()
column_dict = {'shop_id_x' : 'shop_id', 'item_id_x' : 'item_id', 'item_cnt_day' : 'item_cnt_month', 'item_price' : 'avg_price'}
monthly_sales = monthly_sales.rename(columns=column_dict)
 monthly_sales.head()
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows )<load_from_csv>
monthly_sales = monthly_sales.merge(items, on='item_id')
 monthly_sales = monthly_sales.drop(['item_name'], axis=1)
 month = pd.DataFrame([x%12+1 for x in monthly_sales['date_block_num']], columns=['month'])
 year = pd.DataFrame([np.floor(x/12)+2013 for x in monthly_sales['date_block_num']], columns=['year'])
 monthly_sales = pd.concat([monthly_sales, month, year], axis=1)
 monthly_sales = monthly_sales[['ID', 'date_block_num', 'shop_id', 'item_category_id', 'avg_price', 'month', 'year', 'item_cnt_month']]
monthly_sales.head()
bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category )<merge>
def calculate_item_cnt_lagged(df, lag):
tmp = df[['date_block_num', 'ID', 'item_cnt_month']]
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'ID', 'item_cnt_lag'+str(lag)]
shifted.date_block_num = shifted.date_block_num + lag
df = pd.merge(df, shifted, on=['date_block_num', 'ID'], how='left')
 return df
for lag in range(1, 13):
monthly_sales = calculate_item_cnt_lagged(monthly_sales, lag)
 monthly_sales = monthly_sales[monthly_sales['date_block_num'] > 11]
monthly_sales.head()
bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()<define_variables>
id_price = monthly_sales[['ID', 'avg_price']]
x_test_df = test_ids.merge(items, on='item_id')
 x_test_df = x_test_df.merge(id_price, how='left', on='ID')
 x_test_df.insert(loc=2, column='month', value=11)
 x_test_df.insert(loc=3, column='year', value=2015)
 x_test_df.insert(loc=4, column='date_block_num', value=34)
 x_test_df = x_test_df.drop_duplicates()
x_test_df['avg_price'].fillna(x_test_df['avg_price'].mean() , inplace=True)
 monthly_sales_subset = monthly_sales[['ID', 'date_block_num', 'shop_id', 'item_category_id', 'avg_price', 'month', 'year', 'item_cnt_month']]
x_all_df = pd.concat(( x_test_df, monthly_sales_subset))
for lag in range(1, 13):
x_all_df = calculate_item_cnt_lagged(x_all_df, lag)
 x_test_df = x_all_df[x_all_df['date_block_num'] == 34]
x_test_df = x_test_df[['ID', 'shop_id', 'item_category_id', 'avg_price', 'month', 'year',
'item_cnt_lag1', 'item_cnt_lag2', 'item_cnt_lag3', 'item_cnt_lag4',
'item_cnt_lag5', 'item_cnt_lag6', 'item_cnt_lag7', 'item_cnt_lag8',
'item_cnt_lag9', 'item_cnt_lag10', 'item_cnt_lag11', 'item_cnt_lag12']]
x_test_df.head()
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}<feature_engineering>
from scipy.sparse import hstack, vstack
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import OneHotEncoder, normalize
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ] )<merge>
x_train_price = np.array(monthly_sales['avg_price'] ).reshape(-1, 1)
 x_test_price = np.array(x_test_df['avg_price'] ).reshape(-1, 1)
 x_train_price = normalize(x_train_price)
 x_test_price = normalize(x_test_price)
 x_train_lags = np.array(monthly_sales[['item_cnt_lag1', 'item_cnt_lag2', 'item_cnt_lag3', 'item_cnt_lag4',
'item_cnt_lag5', 'item_cnt_lag6', 'item_cnt_lag7', 'item_cnt_lag8',
'item_cnt_lag9', 'item_cnt_lag10', 'item_cnt_lag11', 'item_cnt_lag12']])
 x_test_lags = np.array(x_test_df[['item_cnt_lag1', 'item_cnt_lag2', 'item_cnt_lag3', 'item_cnt_lag4',
'item_cnt_lag5', 'item_cnt_lag6', 'item_cnt_lag7', 'item_cnt_lag8',
'item_cnt_lag9', 'item_cnt_lag10', 'item_cnt_lag11', 'item_cnt_lag12']])
 x_train_categorical = np.array(monthly_sales[['shop_id', 'item_category_id', 'month', 'year']])
 x_test_categorical = np.array(x_test_df[['shop_id', 'item_category_id', 'month', 'year']])
 x_all_categorical = np.concatenate(( x_train_categorical, x_test_categorical))
y_train = np.array(monthly_sales['item_cnt_month'])
 encoder = OneHotEncoder()
encoder.fit(x_all_categorical)
 x_train_categorical = encoder.transform(x_train_categorical)
 x_test_categorical = encoder.transform(x_test_categorical)
 x_train = hstack([x_train_categorical, x_train_price, x_train_lags])
 x_test = hstack([x_test_categorical, x_test_price, x_test_lags])
 y_train = np.clip(y_train, 0, 20)
 print(x_train.shape)
 print(x_test.shape )
active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACT_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLS_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left')
 del closed, closed_agg, bureau
gc.collect()<train_model>
gradient_boost = GradientBoostingRegressor(n_estimators=500)
 gradient_boost.fit(x_train, y_train)
 train_pred = gradient_boost.predict(x_train)
 rmse = np.sqrt(mean_squared_error(y_train, train_pred))
print(f"RMSE on training set: {rmse}" )
<train_model><EOS> 
test_pred = gradient_boost.predict(x_test)
 test_pred = x_test_df.assign(item_cnt_month=test_pred)
 test_pred = test_pred[['ID', 'item_cnt_month']]
test_pred = test_pred.sort_values(by='ID')
 test_pred.to_csv('submission.csv', index=False )