Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_pretrained>
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
def get_data_learner(train_df, train_features, valid_idx,
lr=0.02, epochs=1, layers=[512, 512, 256], ps=[0.2, 0.2, 0.2], name='learner'):
data = TabularDataBunch.from_df(path='.', df=train_df,
dep_var='surface',
valid_idx=valid_idx,
cat_names=[],
cont_names=train_features,
bs=BATCH_SIZE,
procs=[Normalize],
test_df=te)
 learner = tabular_learner(data, layers=layers, ps=ps, metrics=[accuracy], use_bn=True)
 return learner, data<create_dataframe>
def open_csv(file):
path = '/kaggle/input/competitive-data-science-predict-future-sales'
return pd.read_csv(os.path.join(path, file))
items = open_csv('items.csv')
 shops = open_csv('shops.csv')
 item_categories = open_csv('item_categories.csv')
 train = open_csv('sales_train.csv')
 submission = open_csv('sample_submission.csv')
 test = open_csv('test.csv')
learner, data = get_data_learner(tr, features, np.array(valid_idx))<train_model>
attr = ['date', 'date_block_num', 'shop_id', 'item_id', 'item_cnt_day']
old_size = train.shape[0]
train.drop_duplicates(attr, inplace=True)
 print(f'Quantidade de elementos dupliciados: {old_size - train.shape[0]}' )
learner.fit_one_cycle(5, 1e-2 )<train_model>
train.drop(columns=['date'], inplace=True)
 items.drop(columns=['item_name'], inplace=True)
 test.drop(columns=['ID'], inplace=True )
learner.fit_one_cycle(5, 1e-3 )<train_model>
train = pd.merge(train, items, on='item_id', how='left')
 test = pd.merge(test, items, on='item_id', how='left' )
learner.fit_one_cycle(5, 1e-4 )<train_model>
train = reduce_mem_usage(train )
learner.fit_one_cycle(5, 5e-5 )<predict_on_test>
grid = reduce_mem_usage(grid )
val_predictions = np.squeeze(to_np(learner.get_preds(DatasetType.Valid)[0])).argmax(axis=1 )<predict_on_test>
sales_m = train.groupby(index_cols ).agg({'item_cnt_day': 'sum',
'item_price': np.mean} ).reset_index()
print(sales_m )
predictions = np.squeeze(to_np(learner.get_preds(DatasetType.Test)[0])).argmax(axis=1)
 te['surface'] = predictions
te['surface'] = le.inverse_transform(predictions.round().astype(np.int32))<save_to_csv>
sales_m = pd.merge(grid, sales_m, on=index_cols, how='left' ).fillna(0)
 print(f'Quantidade de elementos: {sales_m.shape[0]}
')
 print(sales_m )
te[['surface']].to_csv(f'submission_fastai.csv')
 te[['surface']].head()<count_values>
sales_m = pd.merge(sales_m, items, on='item_id', how='left' )
te['surface'].value_counts()<set_options>
for type_id in ['item_id', 'shop_id', 'item_category_id']:
for column_id, aggregator, aggtype in [('item_price', np.mean, 'avg'), 
('item_cnt_day', np.sum, 'sum'), 
('item_cnt_day', np.mean, 'avg')]:
mean_df = train.groupby([type_id, 'date_block_num'] ).aggregate(aggregator ).reset_index() [[column_id, type_id, 'date_block_num']]
mean_df.columns = [type_id+'_'+aggtype+'_'+column_id, type_id, 'date_block_num']
sales_m = pd.merge(sales_m, mean_df, on=['date_block_num', type_id], how='left')
 print(sales_m.columns )
warnings.filterwarnings("ignore")
 print(os.listdir(".. /input")) 
<load_from_csv>
lag_variables = list(sales_m.columns[6:])+['item_cnt_day']
lags = [1, 2, 3, 4, 5, 6, 12]
def lag_features(df, sales_m):
for lag in lags:
sales_new_df = sales_m.copy()
sales_new_df.date_block_num += lag
sales_new_df = sales_new_df[index_cols+lag_variables]
sales_new_df.columns = index_cols + [f'{lag_feat}_lag_{lag}'
for lag_feat in lag_variables]
df = pd.merge(df, sales_new_df, on=index_cols, how='left')
 return df
lag_train = sales_m.copy()
lag_train = lag_features(lag_train, sales_m)
 print(f'Quantidade de atributos no dataframe: {lag_train.shape[1]}
')
 print(lag_train.columns )
train_df = pd.read_csv(".. /input/X_train.csv", squeeze = True)
 test_df = pd.read_csv(".. /input/X_test.csv" )<train_model>
def fillNaN(df):
for feat in df.columns:
if 'item_cnt' in feat:
df[feat] = df[feat].fillna(0)
 elif 'item_price' in feat:
df[feat] = df[feat].fillna(df[feat].median())
 fillNaN(lag_train )
print("Dimension of train data", train_df.shape[0]//128)
 train_df.head()<load_from_csv>
def date_attr(df):
df['month'] = df['date_block_num'] % 12
df['year'] = df['date_block_num'] // 12
df.drop(columns='date_block_num', inplace=True)
 return df
lag_train = lag_train[lag_train['date_block_num'] > 12]
lag_train = date_attr(lag_train )
y_train = pd.read_csv(".. /input/y_train.csv", skipinitialspace = True, squeeze = True)
 y_train.head()<categorify>
cols_to_drop = lag_variables[:-1] + ['item_price']
lag_train.drop(columns=cols_to_drop, inplace=True)
 lag_train = reduce_mem_usage(lag_train )
y_train["surface_label"] = y_train["surface"].map({'fine_concrete':0, 'concrete':1, 'soft_tiles':2, 'tiled':3, 'soft_pvc':4,
'hard_tiles_large_space':5, 'carpet':6, 'hard_tiles':7, 'wood':8} )<merge>
test['date_block_num'] = 34
test = lag_features(test, sales_m)
 fillNaN(test)
 test = date_attr(test )
train_df = pd.merge(train_df, y_train[["surface_label", "series_id"]], on = "series_id")
 train_df.head()<train_model>
regressor = RandomForestRegressor(n_estimators=10, n_jobs = -1)
 regressor.fit(lag_train.drop(columns=['item_cnt_day']),
lag_train['item_cnt_day'])
 feat_importances = pd.Series(regressor.feature_importances_, index=lag_train.drop(columns=['item_cnt_day'] ).columns)
 feat_importances.nlargest(10 ).plot(kind='barh')
 pred = regressor.predict(test)
 pred = pred.clip(0, 20)
 submission['item_cnt_month'] = pred
submission.to_csv('submission_rf.csv', index=False )
print("Test data dimension",test_df.shape[0]//128)
 test_df.head()<count_values>
param = {'max_depth': 10,
'subsample': 1,
'min_child_weight': 1,
'eta': 0.3,
'num_round': 1000,
'eval_metric': 'rmse',
'verbosity': 0}
<count_duplicates><EOS> 
xgbtrain = xgb.DMatrix(lag_train.drop(columns=['item_cnt_day']),
lag_train['item_cnt_day'])
 bst = xgb.train(param, xgbtrain)
 x = xgb.plot_importance(bst)
 x.figure.set_size_inches(10, 30)
 xgbpredict = xgb.DMatrix(test)
 pred = bst.predict(xgbpredict)
 pred = pred.clip(0, 20)
 submission['item_cnt_month'] = pred
submission.to_csv('submission_xgboost.csv', index=False )