Kaggle competition: Predict Future Sales
train_X = pd.read_csv('.. /input/X_train.csv' ).iloc[:,3:].values.reshape(-1,128,10)
 test_X = pd.read_csv('.. /input/X_test.csv' ).iloc[:,3:].values.reshape(-1,128,10)
 print('train_X shape:', train_X.shape, ', test_X shape:', test_X.shape )<groupby>
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from time import time
import gc
from xgboost import XGBRegressor
train_group = y_train['group_id'].values<compute_test_metric>
LOCATION_KAGGLE = True
SHOW_EDA = True
DOWNSEL_14MONS = False
DOWNSEL_26MONS = True
SHIFT_1MONTH = False
DOWNSEL_OUT = True
USE_CNT_CLIP = False
def sq_dist(a,b):

 return np.sum(( a-b)**2, axis=1)
 def find_run_edges(data, edge):

 if edge == 'left':
border1 = 0
border2 = -1
elif edge == 'right':
border1 = -1
border2 = 0
else:
return False
edge_list = []
linked_list = []
for i in range(len(data)) :
dist_list = sq_dist(data[i, border1, :4], data[:, border2, :4])
 min_dist = np.min(dist_list)
 closest_i = np.argmin(dist_list)
 if closest_i == i:
print('Sample', i, 'linked with itself.Next closest sample used instead.')
 closest_i = np.argsort(dist_list)[1]
dist_list = sq_dist(data[closest_i, border2, :4], data[:, border1, :4])
 rev_dist = np.min(dist_list)
 closest_rev = np.argmin(dist_list)
 if closest_rev == closest_i:
print('Sample', i, '(back-)linked with itself.Next closest sample used instead.')
 closest_rev = np.argsort(dist_list)[1]
if(i != closest_rev):
edge_list.append(i)
 else:
linked_list.append([i, closest_i, min_dist])
 return edge_list, linked_list
def find_runs(data, left_edges, right_edges):

 data_runs = []
for start_point in left_edges:
i = start_point
run_list = [i]
while i not in right_edges:
tmp = np.argmin(sq_dist(data[i, -1, :4], data[:, 0, :4]))
if tmp == i:
tmp = np.argsort(sq_dist(data[i, -1, :4], data[:, 0, :4])) [1]
i = tmp
run_list.append(i)
 data_runs.append(np.array(run_list))
return data_runs<find_best_params>
RANDOM_SEED = 730
np.random.seed(RANDOM_SEED)
 print(RANDOM_SEED )
train_left_edges, train_left_linked = find_run_edges(train_X, edge='left')
 train_right_edges, train_right_linked = find_run_edges(train_X, edge='right')
 print('Found', len(train_left_edges), 'left edges and', len(train_right_edges), 'right edges.' )<compute_test_metric>
data_dir = ".. /input/competitive-data-science-predict-future-sales/"
trans_file = "sales_train.csv"
out_file = "test.csv"
items_file = "items.csv"
if LOCATION_KAGGLE:
categs_file = "item_categories.csv"
shops_file = "shops.csv"
else:
categs_file = "item_categories-translated.csv"
shops_file = "shops-translated.csv"
item_names_file = "items-translated.csv"
train_runs = find_runs(train_X, train_left_edges, train_right_edges )<save_to_csv>
df_trans = pd.read_csv(data_dir+trans_file)
 if SHIFT_1MONTH:
df_trans['date_block_num'] += 1
df_out = pd.read_csv(data_dir+out_file)
 df_items = pd.read_csv(data_dir+items_file)
 df_categs = pd.read_csv(data_dir+categs_file)
 df_shops = pd.read_csv(data_dir+shops_file)
 if LOCATION_KAGGLE == False:
df_item_names = pd.read_csv(data_dir+item_names_file)
 df_items['item_name'] = df_item_names['item_name']
y_train['run_id'] = 0
y_train['run_pos'] = 0
for run_id in range(len(train_runs)) :
for run_pos in range(len(train_runs[run_id])) :
series_id = train_runs[run_id][run_pos]
y_train.at[ series_id, 'run_id' ] = run_id
y_train.at[ series_id, 'run_pos' ] = run_pos
y_train.to_csv('y_train_with_runs.csv', index=False)
 y_train.tail()<count_unique_values>
num_months = len(df_trans['date_block_num'].unique())
 print("
Full Transactions has {} entries covering {} months.
 ".format(len(df_trans),num_months))
print("The number of unique values for each column:")
 df_trans.nunique()
flat_list = [series_id for run in train_runs for series_id in run]
print(len(flat_list), len(np.unique(flat_list)) )<count_unique_values>
surface_cnt_train = []
for i in range(len(train_runs)) :
surface_cnt_train.append(len(np.unique(y_train['surface'].iloc[train_runs[i]])) )<define_variables>
print('Id of runs with different surfaces:')
 for i in range(len(surface_cnt_train)) :
if surface_cnt_train[i] > 1:
print(str(i)+ ', number of surfaces: ' + str(surface_cnt_train[i]))<merge>
if SHIFT_1MONTH:
print(" - - - SHIFT_1MONTH - - -
Finding all future_items")
 out_items = np.sort(df_out.item_id.unique())
 num_out_items = len(out_items)
 print("
Number of items in Output: {}".format(num_out_items))
trans_items = np.sort(df_trans.item_id.unique())
 print("Number of items in FULL Transactions: {}".format(len(trans_items)))
 future_items = []
for item in out_items:
if item not in trans_items:
future_items.append(item)
 if len(future_items)> 0:
print("There are {} future_items which are missing from the FULL Transactions data:".format(
 len(future_items)) ,"
")
 else:
print("
All of the Test Items are in the Train data.
 ")
 plt.hist(future_items, bins=50)
 plt.xlabel("item_id")
 plt.title("Future Items missing from the FULL Transactions")
 plt.show()
X_train = pd.merge(y_train[['series_id', 'run_id']], X_train, on='series_id')
 X_train.head()<load_from_csv>
if SHIFT_1MONTH and LOCATION_KAGGLE:
df_month = df_trans[df_trans['date_block_num'] == 34].copy()
df_val = df_out.copy()
out_vals = []
for iout in df_val.index:
this_shop = df_val.loc[iout,'shop_id']
this_item = df_val.loc[iout,'item_id']
out_vals.append(df_month.loc[(df_month['shop_id'] == this_shop)& 
(df_month['item_id'] == this_item),'item_cnt_day'].sum())
 df_val['item_cnt_month'] = np.array(out_vals)
 clip_level = 20
num_over20 = sum(df_val['item_cnt_month'] > clip_level)
 print("
Check for over-20 values, found/clipped {} of them.".format(num_over20))
if num_over20 > 0:
df_val.loc[df_val['item_cnt_month'] > clip_level, 'item_cnt_month'] = clip_level
print("
Sum of output item_cnt_month: {:.0f}, average count-per-month per shop-item: {:.3f}
".format(
 df_val['item_cnt_month'].sum() , df_val['item_cnt_month'].sum() /(len(df_val))))
df_val[['ID','item_cnt_month']].to_csv("submission_Oct15.csv", index=None)
 del df_month, df_val, out_vals
gc.collect()
test_X = pd.read_csv('.. /input/X_test.csv' ).iloc[:,3:].values.reshape(-1,128,10 )<find_best_params>
if DOWNSEL_14MONS:
df_trans =(df_trans[(( df_trans['date_block_num'] >= 20)& 
(df_trans['date_block_num'] <= 33)) ] ).reset_index().drop('index',axis=1)
 elif DOWNSEL_26MONS:
df_trans =(df_trans[(( df_trans['date_block_num'] >= 8)& 
(df_trans['date_block_num'] <= 33)) ] ).reset_index().drop('index',axis=1)
 else:
df_trans =(df_trans[(( df_trans['date_block_num'] <= 33)) ] ).reset_index().drop('index',axis=1)
 num_months = len(df_trans['date_block_num'].unique())
 print("Using {} months of data.".format(num_months))
test_left_edges, test_left_linked = find_run_edges(test_X, edge='left')
 test_right_edges, test_right_linked = find_run_edges(test_X, edge='right')
 print('Found', len(test_left_edges), 'left edges and', len(test_right_edges), 'right edges.' )<compute_test_metric>
df_trans['item_cnt_clip'] = df_trans['item_cnt_day']
df_trans.loc[df_trans['item_cnt_clip'] > 20, 'item_cnt_clip'] = 20
test_runs = find_runs(test_X, test_left_edges, test_right_edges )<count_unique_values>
df_trans = pd.merge(df_trans, df_items[['item_id','item_category_id']].set_index('item_id'), left_on='item_id',
right_index=True, how='left', sort=False)
 df_out = pd.merge(df_out, df_items[['item_id','item_category_id']].set_index('item_id'), left_on='item_id',
right_index=True, how='left', sort=False )
flat_list = [series_id for run in test_runs for series_id in run]
print(len(flat_list), len(np.unique(flat_list)) )<define_variables>
df_trans['log10_price'] = np.log10(np.abs(df_trans['item_price']))
print("Min and Max of the log10 of item price: {:.2f} , {:.2f}".format(df_trans['log10_price'].min() ,
df_trans['log10_price'].max()))
df_trans['item_price3'] =(( df_trans['item_price'] > 300.00 ).astype(int)+ 
(df_trans['item_price'] > 1000.00 ).astype(int))
lost_samples = np.array([ i for i in range(len(test_X)) if i not in np.concatenate(test_runs)])
 print(lost_samples)
 print(len(lost_samples))<sort_values>
df_trans['date_dt'] = pd.to_datetime(df_trans.date, format='%d.%m.%Y')
 df_trans['day'] = df_trans.date_dt.dt.day
df_trans['month'] = df_trans.date_dt.dt.month
df_trans['year'] = df_trans.date_dt.dt.year
find_run_edges(test_X[lost_samples], edge='left')[1][0]<filter>
print("The selected Transactions have {} entries covering {} months.".format(len(df_trans),num_months))
print("The number of unique values for each column:")
 df_trans.nunique()
lost_run = np.array(lost_samples[find_runs(test_X[lost_samples], [0], [5])[0]])
 test_runs.append(lost_run )<load_from_csv>
df_out.tail(5 )
X_test = pd.read_csv(".. /input/X_test.csv" )<prepare_output>
if SHIFT_1MONTH:
print(df_shops.loc[36])
 missing_shops = [36]
y_test = pd.DataFrame(columns = ['series_id', 'run_id', 'run_pos'])
 y_test['series_id'] = X_test.series_id.unique()<feature_engineering>
out_cats = np.sort(df_out.item_category_id.unique())
 num_out_cats = len(out_cats)
 print("
Number of categories appearing in Output: {}".format(num_out_cats))
print(out_cats)
 trans_cats = np.sort(df_trans.item_category_id.unique())
 print("
Number of categories in Transactions: {}".format(len(trans_cats)))
 print(trans_cats)
 print("")
 missing_out = False
missing_categories = []
for iout in out_cats:
if iout not in trans_cats:
print("Missing Output category: {}".format(iout))
missing_categories.append(iout)
 missing_out = True
if missing_out:
print("
Some Output categories are missing from the Transactions data.")
 else:
print("
All of the Output categories are in the Transaction data.
 " )
y_test['run_id'] = 0
y_test['run_pos'] = 0
for run_id in range(len(test_runs)) :
for run_pos in range(len(test_runs[run_id])) :
series_id = test_runs[run_id][run_pos]
y_test.at[ series_id, 'run_id' ] = run_id
y_test.at[ series_id, 'run_pos' ] = run_pos
y_test.head()<merge>
df_categs.loc[missing_categories]
X_test = pd.merge(y_test[['series_id', 'run_id']], X_test, on='series_id')
 X_test.head()<normalization>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.degrees(math.atan2(t0, t1))
t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.degrees(math.asin(t2))
t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.degrees(math.atan2(t3, t4))
return X, Y, Z<feature_engineering>
out_items = np.sort(df_out.item_id.unique())
 num_out_items = len(out_items)
 print("
Number of items in Output: {}".format(num_out_items))
print("
Output has {} items x {} shops = {} pairs;
which agrees with the length of df_out = {}".format(
 num_out_items, num_out_shops, num_out_shops * num_out_items, len(df_out)))
 trans_items = np.sort(df_trans.item_id.unique())
 print("
Number of items in Transactions: {}".format(len(trans_items)))
 print("")
 missing_items = []
for item in out_items:
if item not in trans_items:
missing_items.append(item)
 if len(missing_items)> 0:
print("Some Output Items, {} of them, are missing from the Transactions data:".format(
 len(missing_items)) ," 
(However, these all have known",
"categories,", "except for 5441 and 6439)
 ")
 if SHOW_EDA:
print(missing_items)
 else:
print("
All of the Test Items are in the Train data.
 ")
 plt.hist(missing_items, bins=50)
 plt.xlabel("item_id")
 plt.title("Output items that are missing from Transactions")
 plt.show()
def fe_step0(data):
data['norm_quat'] = data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2 + data['orientation_W']**2
data['mod_quat'] =(data['norm_quat'])**0.5
data['norm_X'] = data['orientation_X'] / data['mod_quat']
data['norm_Y'] = data['orientation_Y'] / data['mod_quat']
data['norm_Z'] = data['orientation_Z'] / data['mod_quat']
data['norm_W'] = data['orientation_W'] / data['mod_quat']
return data<train_model>
X_train = fe_step0(X_train)
 X_test = fe_step0(X_test)
 print(X_train.shape)
 X_train.head(5 )<categorify>
trans_sum_cnt = df_trans['item_cnt_day'].sum()
trans_sum_clip = df_trans['item_cnt_clip'].sum()
print("
Total count = Sum of all item_cnt_day is: {}(clipped ~ {})".format(trans_sum_cnt,trans_sum_clip))
average_totcnt_month = trans_sum_cnt/num_months
average_totcnt_clip = trans_sum_clip/num_months
print("
Average Total-count /month is: {:.3f}(clipped ~ {:.3f})
 ".format(average_totcnt_month,average_totcnt_clip))
def fe_step1(data):

 x, y, z, w = data['norm_X'].tolist() , data['norm_Y'].tolist() , data['norm_Z'].tolist() , data['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 data['euler_x'] = nx
data['euler_y'] = ny
data['euler_z'] = nz
return data<train_model>
X_train = fe_step1(X_train)
 X_test = fe_step1(X_test)
 print(X_train.shape)
 X_train.head(5 )<feature_engineering>
its_missing = []
for iit in df_items.index:
its_missing.append(df_items.loc[iit,'item_id'] in missing_items)
 newhot20_items = df_items[(df_items['item_category_id'] == 20)& its_missing].item_id.values
newhot28_items = df_items[(df_items['item_category_id'] == 28)& its_missing].item_id.values
print("
Missing items that are in Cats 20 or 28:")
 print(newhot28_items,"
",newhot20_items)
def feat_eng(data, test_data=False):

 df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
if test_data:
col_for_group_by = ['run_id']
else:
col_for_group_by = ['run_id', 'surface']
for col in data.columns:
if col in ['row_id','series_id','measurement_number', 'group_id', 'run_id', 'run_pos', 'surface']:
continue
df[col + '_mean'] = data.groupby(col_for_group_by)[col].mean()
df[col + '_median'] = data.groupby(col_for_group_by)[col].median()
df[col + '_max'] = data.groupby(col_for_group_by)[col].max()
df[col + '_min'] = data.groupby(col_for_group_by)[col].min()
df[col + '_std'] = data.groupby(col_for_group_by)[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(col_for_group_by)[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mean_change_of_abs_change'] = data.groupby(col_for_group_by)[col].apply(mean_change_of_abs_change)
 df[col + '_abs_max'] = data.groupby(col_for_group_by)[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(col_for_group_by)[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df<train_model>
its_missing = []
for iit in df_items.index:
its_missing.append(df_items.loc[iit,'item_id'] in missing_items)
 newhot37_items = df_items[(df_items['item_category_id'] == 37)& its_missing].item_id.values
newhot40_items = df_items[(df_items['item_category_id'] == 40)& its_missing].item_id.values
newhot55_items = df_items[(df_items['item_category_id'] == 55)& its_missing].item_id.values
print("
Missing items that are in Cats 37, 40, 55:")
 print(newhot37_items,"
",newhot40_items,"
",newhot55_items )
X_train = feat_eng(X_train)
 X_test = feat_eng(X_test, test_data=True)
 print("New features: ",X_train.shape )<concatenate>
df_categs.loc[[9,20,28,47]]
its_missing = []
for iit in df_items.index:
its_missing.append(df_items.loc[iit,'item_id'] in missing_items)
 newhot47_items = df_items[(df_items['item_category_id'] == 47)& its_missing].item_id.values
print("
Missing items that are in Cat 47:")
 print(newhot47_items)
feature_list.extend(['totl_anglr_vel', 'totl_linr_acc', 'acc_vs_vel'] )<merge>
item_cnt_col = 'item_cnt_day'
if USE_CNT_CLIP:
item_cnt_col = 'item_cnt_clip'
totcnt_vs_month = df_trans[['date_block_num',item_cnt_col]].groupby('date_block_num' ).agg(sum)
 if USE_CNT_CLIP:
sel_totcnt_month = average_totcnt_clip
else:
sel_totcnt_month = average_totcnt_month
nov15_oct15_ratio = totcnt_vs_month.loc[22,item_cnt_col]/totcnt_vs_month.loc[21,item_cnt_col]
nov15_totcnt = totcnt_vs_month.loc[33,item_cnt_col] * nov15_oct15_ratio
nov15_ave_ratio = nov15_totcnt / sel_totcnt_month
dec15_totcnt =(totcnt_vs_month.loc[33,item_cnt_col]*
totcnt_vs_month.loc[23,item_cnt_col]/totcnt_vs_month.loc[21,item_cnt_col])
 ax = df_trans[['date_block_num',item_cnt_col]].groupby('date_block_num' ).agg(sum ).plot(
 kind='line',marker='o',figsize=(8,5))
plt.plot([34.0,34.0],[sel_totcnt_month,nov15_totcnt],color='gray')
 plt.plot([34],[nov15_totcnt],color='red',marker="*",markersize=15)
 plt.plot([35.0,35.0],[sel_totcnt_month,dec15_totcnt],color='green')
 plt.plot([35],[dec15_totcnt],color='green',marker="*",markersize=15)
 plt.plot([22],[totcnt_vs_month.loc[22,item_cnt_col]],color='red',marker="*",markersize=15)
 plt.plot([23],[totcnt_vs_month.loc[23,item_cnt_col]],color='green',marker="*",markersize=15)
 min_date_block = df_trans.date_block_num.min()
plt.plot([min_date_block,36],[sel_totcnt_month, sel_totcnt_month],color='gray')
 plt.ylim(0,)
 plt.xlim(min_date_block-1,36)
 plt.title("Total counts in each month.Stars: Nov(red), Dec(green)")
 plt.show()
print("Average line is at {:.1f}".format(sel_totcnt_month))
print("Nov'15 to average ratio is {:.4f}".format(nov15_ave_ratio))
print("Nov'15 to Oct'15 ratio is {:.4f}".format(nov15_oct15_ratio))
print("November 2015 total cnt ~ {:.1f}".format(nov15_totcnt))
X_train = pd.merge(y_train[['series_id', 'run_id', 'surface']], X_train, on=['run_id', 'surface'])
 X_train.drop(['run_id', 'surface'], axis=1, inplace=True)
 X_train.head()<merge>
if False:
predict_avecnt = 0.265
print("
Using a manual-provided constant value of {:.3f}
".format(predict_avecnt))
df_out['item_cnt_month'] = predict_avecnt
if False:
if USE_CNT_CLIP:
predict_avecnt = nov15_totcnt /(len(out_shops)*len(out_items))
print("
Using a Nov'15 constant value of {:.3f}
".format(predict_avecnt))
df_out['item_cnt_month'] = predict_avecnt
else:
print("Best to set USE_CLIP_CNT = True for this estimate.")
X_test = pd.merge(y_test[['series_id', 'run_id']], X_test, on='run_id')
 X_test.drop(['run_id'], axis=1, inplace=True)
 X_test.head()<groupby>
if False:
using_month = 10
using_year = 2015
if USE_CNT_CLIP:
print("
* Warning: better to have USE_CNT_CLIP = False for this.
 ")
 df_month = df_trans[(df_trans['year'] == using_year)&(df_trans['month'] == using_month)].copy()
out_vals = []
for iout in df_out.index:
this_shop = df_out.loc[iout,'shop_id']
this_item = df_out.loc[iout,'item_id']
out_vals.append(df_month.loc[(df_month['shop_id'] == this_shop)& 
(df_month['item_id'] == this_item),'item_cnt_day'].sum())
 print("
Unscaled cnt data from month {} of year {}.
 ".format(using_month, using_year))
df_out['item_cnt_month'] = np.array(out_vals )
y_gr = y_train[['group_id','surface','series_id']].groupby(['group_id','surface'] ).count().reset_index()
y_gr.rename(columns={'series_id':'cnt'}, inplace=True)
 y_gr.head()<split>
if False:
if USE_CNT_CLIP:
print("
* Warning: better to have USE_CNT_CLIP = False for this.
 ")
 print("
Averaging counts over {} months.
 ".format(num_months))
out_vals = []
for iout in df_out.index:
this_shop = df_out.loc[iout,'shop_id']
this_item = df_out.loc[iout,'item_id']
out_vals.append(df_trans.loc[(df_trans['shop_id'] == this_shop)& 
(df_trans['item_id'] == this_item),'item_cnt_day'].sum() /num_months)
 scale_factor = 0.515
df_out['item_cnt_month'] = scale_factor * np.array(out_vals)
 print("Scaled counts by {}".format(scale_factor))
y_gr_temp = y_gr[y_gr.surface!='hard_tiles']
y_gr_train, y_gr_val = train_test_split(y_gr_temp, test_size=0.15, random_state=11, stratify=y_gr_temp.surface)
 y_gr_train = y_gr_train.append(y_gr[y_gr.surface=='hard_tiles'] )<groupby>
ShopCatIt = True
if ShopCatIt:
if True:
select_sc =(( df_trans['date_block_num'] >= 31)&(df_trans['date_block_num'] <= 33))
select_ic =(( df_trans['date_block_num'] >= 31)&(df_trans['date_block_num'] <= 33))
else:
select_sc = df_trans['date_block_num'] == 33
select_ic = df_trans['date_block_num'] == 33
shop_cat_ctab = pd.crosstab(df_trans.loc[select_sc,'shop_id'],df_trans.loc[select_sc,'item_category_id'],
values=df_trans.loc[select_sc,'item_cnt_day'], aggfunc=sum,
normalize='all' ).fillna(0.0)
 item_cat_ctab = pd.crosstab(df_trans.loc[select_ic,'item_id'],df_trans.loc[select_ic,'item_category_id'],
values=df_trans.loc[select_ic,'item_cnt_day'], aggfunc=sum,
normalize='columns' ).fillna(0.0 )
y_gr_val[['group_id', 'surface']].groupby('surface' ).count()<groupby>
if ShopCatIt:
this_ID = 6004
this_shop = df_out.loc[this_ID,'shop_id']
this_cat = df_out.loc[this_ID,'item_category_id']
shop_frac = shop_cat_ctab.T.apply(np.sum)[this_shop]
cat_frac = shop_cat_ctab.apply(np.sum)[this_cat]
print("Actual ctab: {} Approx product: {}".format(shop_cat_ctab.loc[this_shop, this_cat],shop_frac*cat_frac))
y_gr_train[['group_id', 'surface']].groupby('surface' ).count()<merge>
if ShopCatIt:
df_out['shop_frac'] = df_out['shop_id'].map(shops_fracs)
 df_out['cat_frac'] = df_out['item_category_id'].map(cats_fracs)
 df_out.fillna(0.0, inplace=True)
y_train_data = y_train.merge(y_gr_train.group_id, on='group_id' )<merge>
if ShopCatIt:
df_out['item_frac'] = 0.0
sc_categories = list(shop_cat_ctab.columns)
 ic_categories = list(item_cat_ctab.columns)
 for this_cat in df_out['item_category_id'].unique() :
if(this_cat in ic_categories):
items_fracs =(item_cat_ctab[this_cat]/sum(item_cat_ctab[this_cat]))
select = df_out['item_category_id'] == this_cat
df_out.loc[select,'item_frac'] = df_out.loc[select,'item_id'].map(items_fracs)
 if(this_cat in sc_categories):
df_out.loc[select,'shopcat_frac'] = df_out.loc[select,'shop_id'].map(shop_cat_ctab[this_cat])
 df_out.fillna(0.0, inplace=True)
y_val_data = y_train.merge(y_gr_val.group_id, on='group_id' )<groupby>
current_missing_sum = sum(df_out.set_index('item_id' ).loc[missing_items,'item_cnt_month'])
 num_missing_itshops = len(missing_items)* num_out_shops
if True:
print("
The sum of the missing items is {:.2f}, giving an average of {:.3f} per item-shop
".format(
 current_missing_sum, current_missing_sum/num_missing_itshops))
missing_value = 0.475
if SHIFT_1MONTH:
missing_value = 0.479
for imiss in range(len(missing_items)) :
df_out.loc[df_out['item_id'] == missing_items[imiss] ,'item_cnt_month'] = missing_value
print("Set {} missing items to {}".format(len(missing_items), missing_value))
newhot20_value = 3.60
newhot28_value = 6.6
newhot47_value = 0.530
if SHIFT_1MONTH:
newhot20_value = 3.76
newhot28_value = 6.79
newhot47_value = 1.19
for imiss in range(len(newhot20_items)) :
df_out.loc[df_out['item_id'] == newhot20_items[imiss] ,'item_cnt_month'] = newhot20_value
print("Set {} newhot-20 items to {}".format(len(newhot20_items), newhot20_value))
for imiss in range(len(newhot28_items)) :
df_out.loc[df_out['item_id'] == newhot28_items[imiss] ,'item_cnt_month'] = newhot28_value
print("Set {} newhot-28 items to {}".format(len(newhot28_items), newhot28_value))
for imiss in range(len(newhot47_items)) :
df_out.loc[df_out['item_id'] == newhot47_items[imiss] ,'item_cnt_month'] = newhot47_value
print("Set {} newhot-47 items to {}".format(len(newhot47_items), newhot47_value))
newhot37_value = 1.8
newhot40_value = 0.30
newhot55_value = 0.85
if SHIFT_1MONTH:
newhot37_value = 1.81
newhot40_value = 0.95
newhot55_value = 0.62
for imiss in range(len(newhot37_items)) :
df_out.loc[df_out['item_id'] == newhot37_items[imiss] ,'item_cnt_month'] = newhot37_value
print("Set {} newhot-37 items to {}".format(len(newhot37_items), newhot37_value))
for imiss in range(len(newhot40_items)) :
df_out.loc[df_out['item_id'] == newhot40_items[imiss] ,'item_cnt_month'] = newhot40_value
print("Set {} newhot-40 items to {}".format(len(newhot40_items), newhot40_value))
for imiss in range(len(newhot55_items)) :
df_out.loc[df_out['item_id'] == newhot55_items[imiss] ,'item_cnt_month'] = newhot55_value
print("Set {} newhot-55 items to {}".format(len(newhot55_items), newhot55_value))
cnt_5441 = 0.0
cnt_6439 = 0.0
df_out.loc[df_out['item_id'] == 5441 ,'item_cnt_month'] = cnt_5441
df_out.loc[df_out['item_id'] == 6439 ,'item_cnt_month'] = cnt_6439
print("Set 1 item 5541 to {}".format(cnt_5441))
print("Set 1 item 6439 to {}".format(cnt_6439))
if ShopCatIt:
shop_modulation = 0.34
if SHIFT_1MONTH:
shop_modulation = 0.8
for imiss in range(len(missing_items)) :
select =(df_out['item_id'] == missing_items[imiss])
 df_out.loc[select, 'item_cnt_month'] =(df_out.loc[select, 'item_cnt_month'] * 
(1.0 - shop_modulation + shop_modulation*42.0*df_out.loc[select, 'shop_frac']))
print("Scaled all {} missing items by shop_frac with modulation = {:.3f}.".format(
 len(missing_items),shop_modulation))
current_missing_sum = sum(df_out.set_index('item_id' ).loc[missing_items,'item_cnt_month'])
 else:
print("The {} 'missing items' were left as is.".format(len(missing_items)))
 if SHIFT_1MONTH:
for imiss in range(len(future_items)) :
df_out.loc[df_out['item_id'] == future_items[imiss] ,'item_cnt_month'] = 0.0
print("
* Set {} Future items to {} *".format(len(future_items), 0.0))
print(" Leaving {} missing-minus-future items.".format(len(missing_items)- len(future_items)))
 current_missing_sum = sum(df_out.set_index('item_id' ).loc[missing_items,'item_cnt_month'])
 num_missing_itshops =(len(missing_items)- len(future_items)) * num_out_shops
print("The sum of the missing-minus-future items is {:.2f}, giving an average of {:.3f} per item-shop".format(
 current_missing_sum, current_missing_sum/num_missing_itshops))
else:
print("
The sum of the missing items is {:.2f}, giving an average of {:.3f} per item-shop".format(
 current_missing_sum, current_missing_sum/num_missing_itshops))
y_train_data[['series_id', 'surface']].groupby('surface' ).count()<groupby>
clip_level = 20
num_over20 = sum(df_out['item_cnt_month'] > clip_level)
 print("
Check for over-20 values, found/clipped {} of them.".format(num_over20))
if num_over20 > 0:
df_out.loc[df_out['item_cnt_month'] > clip_level, 'item_cnt_month'] = clip_level
print("
Sum of output item_cnt_month: {:.0f}, average count-per-month per shop-item: {:.3f}
".format(
 df_out['item_cnt_month'].sum() , df_out['item_cnt_month'].sum() /(len(df_out))))
y_val_data[['group_id', 'surface']].groupby('surface' ).count()<merge>
df_out[['ID','item_cnt_month']].to_csv("submission.csv", index=None)
X = y_train_data.merge(X_train, on='series_id')
 X = X.drop(['group_id', 'run_id', 'run_pos', 'series_id', 'surface'], axis=1)
 X = np.array(X)
 X_val = y_val_data.merge(X_train, on='series_id')
 X_val = X_val.drop(['group_id', 'run_id', 'run_pos', 'series_id', 'surface'], axis=1)
 X_val = np.array(X_val )<sort_values>
if True:
version = 'current'
if version == 'current':
df_sub = pd.read_csv("submission.csv")
 else:
df_sub = pd.read_csv(".. /submissions/submission_"+version+".csv")
 print("
Sum of output item_cnt_month: {:.0f}, average count-per-month per shop-item: {:.3f}
".format(
 df_sub['item_cnt_month'].sum() , df_sub['item_cnt_month'].sum() /(len(df_sub))))
rmse_vs_0 = np.sqrt(sum(df_sub['item_cnt_month']*df_sub['item_cnt_month'])/len(df_sub))
print("RMSE_vs_0 is {:.5f} for version = {}".format(rmse_vs_0,version))
df_sub['int_cnt_month'] =(0.50 + df_sub['item_cnt_month'] ).astype(int)
X_test = X_test.sort_values('series_id')
 X_t = X_test.drop(['series_id'], axis=1)
 X_t = np.array(X_t )<define_variables>
if SHIFT_1MONTH:
newhot_cats = df_missing.loc[df_missing['newhot'] == 1, 'item_category_id'].unique()
print("Number of new-hot missing categories: {}
".format(len(newhot_cats)))
 for this_cat in newhot_cats:
print(df_categs.loc[this_cat, 'item_category_name'])
n_featurs = X.shape[1]
print(n_featurs )<choose_model_class>
if SHIFT_1MONTH:
newhot_items = df_missing.loc[df_missing['newhot'] == 1, 'item_id'].unique()
print("Number of new-hot missing items: {}
".format(len(newhot_items)))
 for this_item in newhot_items:
print(df_items.loc[this_item, 'item_name'])
rf_clf = RandomForestClassifier(n_estimators=100, max_depth=11, verbose=1,
class_weight='balanced',
random_state=11, n_jobs=10 )<train_model>
if SHIFT_1MONTH:
boring_items = df_missing.loc[(df_missing['boring'] == 1), 'item_id'].unique()
print("Number of boring missing items: {}
".format(len(boring_items)))