Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<save_to_csv>
MODEL_VERSION = 'v14'
submission.to_csv('submission.csv', index=False)
 !ls<define_variables>
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 60
END_DAY = 1913
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 TRAIN_SPLIT = '2016-03-27'
EVAL_SPLIT = '2016-04-24'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
TASK_TYPE='CPU'
TARGET = 'trip_duration'<load_from_csv>
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32'
}
df = pd.read_csv('.. /input/train.csv');
df = df[df.trip_duration < 3600]
df.head()<compute_test_metric>
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_validation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation
def haversine_array(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h<compute_test_metric>
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['id', 'sales']].groupby('id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'rmean_{lag_col}_{window}'
strain[window_col] = strain[['id', lag_col]].groupby('id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales for done for window {window}')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday"
}
for date_feat_name, date_feat_func in date_features.items() :
if not date_feat_name in dt.columns:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )
def bearing_array(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))<create_dataframe>
%%time
sales_train_validation = load_data()
def drop_odd_values(dataframe):
dataframe['passenger_count'] = dataframe.passenger_count.map(lambda x: 1 if x == 0 else x)
 dataframe = dataframe[dataframe.passenger_count <= 6]
dataframe = dataframe[dataframe.haversine_distance <= 90] 
<feature_engineering>
sales_train_validation["sale"] =(( sales_train_validation['sell_price'] * 100 % 10)< 6 ).astype('int8' )
def prep_dataset(dataframe, target='trip_duration'):
dataframe['datetime'] = pd.to_datetime(dataframe['pickup_datetime'])
 dataframe['year'] = dataframe['datetime'].dt.year
dataframe['month'] = dataframe['datetime'].dt.month
dataframe['day'] = dataframe['datetime'].dt.day
dataframe['weekday'] = dataframe['datetime'].dt.weekday + 1
dataframe['hour'] = dataframe['datetime'].dt.hour
dataframe['haversine_distance'] = dataframe.apply(lambda x: haversine_array(x['pickup_latitude'], x['pickup_longitude'], x['dropoff_latitude'], x['dropoff_longitude']), axis=1)
 dataframe['bearing'] = dataframe.apply(lambda x: bearing_array(x['pickup_latitude'], x['pickup_longitude'], x['dropoff_latitude'], x['dropoff_longitude']),axis=1)
 dataframe.loc[:, 'center_latitude'] =(dataframe['pickup_latitude'].values + dataframe['dropoff_latitude'].values)/ 2
dataframe.loc[:, 'center_longitude'] =(dataframe['pickup_longitude'].values + dataframe['dropoff_longitude'].values)/ 2
selected_columns = ['vendor_id', 'passenger_count', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'day', 'weekday', 'hour', 'month', 'haversine_distance', 'center_latitude', 'center_longitude', 'bearing']
return selected_columns<prepare_x_and_y>
%%time
make_lag_features(sales_train_validation)
 make_date_features(sales_train_validation )
X_train = df[prep_dataset(df)]
y_train = df[TARGET]
df['passenger_count'] = df.passenger_count.map(lambda x: 1 if x == 0 else x)
 df = df[df.passenger_count <= 6]
df = df[df.haversine_distance <= 90]
X_train.shape, y_train.shape<randomize_order>
total = len(sales_train_validation['event_type_2'])
 zero = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 0])
 one = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 1])
 two = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 2])
 print('0: '+ str(zero))
print('1: ' + str(one))
print('2: ' + str(two))
print('-'*7 + '
Total: ' + str(total))
del total; del zero; del one; del two
cv = ShuffleSplit(n_splits=4, test_size=0.1, train_size=0.2, random_state=0 )<compute_train_metric>
sales_train_validation.isnull().sum()
rf = RandomForestRegressor()
times = -cross_val_score(rf, X_train, y_train, cv=cv, scoring='neg_mean_squared_log_error')
 time = [np.sqrt(l)for l in times]
time[:5]
np.mean(time)
<train_model>
before = len(sales_train_validation)
 sales_train_validation.dropna(inplace = True)
 after = len(sales_train_validation)
 print(f"Reduced {(before-after)/before}%")
rf.fit(X_train, y_train )<load_from_csv>
print("Splitting data into train, validation, evaluation set")
 train = sales_train_validation[(sales_train_validation['date'] >= BEGIN_DATE)&(sales_train_validation['date'] <= TRAIN_SPLIT)]
validation = sales_train_validation[(sales_train_validation['date'] > TRAIN_SPLIT)&(sales_train_validation['date'] <= EVAL_SPLIT)]
evaluation = sales_train_validation[(sales_train_validation['date'] > EVAL_SPLIT)]
del sales_train_validation; gc.collect()
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()<create_dataframe>
print(train["cat_id"].value_counts())
 print(validation["cat_id"].value_counts() )
X_test = df_test[prep_dataset(df_test)]<predict_on_test>
categorical_features = [
'item_id', 'dept_id', 'store_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]
y_pred = rf.predict(X_test)
 y_pred.mean()<load_from_csv>
drop = ["id", "date", "sales", "day", "wm_yr_wk", "weekday", "cat_id"]
train_columns = train.columns[~train.columns.isin(drop)]
print(train_columns )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col=0)
 submission.head()<prepare_output>
train_sets = [train[train["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del train; gc.collect()
submission['trip_duration'] = y_pred
submission.head()<save_to_csv>
val_sets = [validation[validation["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del validation; gc.collect()
submission.to_csv('submission_taxi.csv' )<set_options>
train_pools = [lgb.Dataset(
 data=train_sets[category][train_columns],
label=train_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del train_sets; gc.collect()
%matplotlib inline<load_from_csv>
val_pools = [lgb.Dataset(
 data=val_sets[category][train_columns],
label=val_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del val_sets; del evaluation; gc.collect()
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
if TEST_RUN:
ITERATIONS=1
else:
ITERATIONS = 1200
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : ITERATIONS,
'num_leaves': 128,
"min_data_in_leaf": 100,
"early_stopping_round": 50
}
df_train.isna().sum()<count_missing_values>
%%time
models = [lgb.train(params,
train_pools[category],
valid_sets = val_pools[category],
verbose_eval=20)
 for category in range(3)]
for cat,model in enumerate(models):
model.save_model(f"model_cat{cat}_{MODEL_VERSION}.lgb" )
df_train.isnull().sum() 
<count_duplicates>
del train_pools, val_pools; gc.collect()
df_train.duplicated().sum() 
<feature_engineering>
df = load_data(train=False)
 make_date_features(df)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8' )
df_train = df_train[(df_train['trip_duration'] > 60)&(df_train['trip_duration'] < 4600)]
df_train['trip_duration'] = np.log(df_train['trip_duration'].values)
<filter>
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'id'])
 dt.loc[dt['date'] == str(day),f'rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print("Lag features done")
 print(type(dt))
<feature_engineering>
%%time

 END_DATE = EVAL_SPLIT
ZERO_THRESHOLD = 0.01
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 for i in range(3):
prediction_data_cat = prediction_data.loc[(prediction_data['date'] == str(pred_date)) &(prediction_data["cat_id"] == i), train_columns]
prediction = models[i].predict(prediction_data_cat)
 df.loc[(df['date'] == str(pred_date)) &(df.index.isin(prediction_data_cat.index)) , 'sales'] = prediction
<data_type_conversions>
del prediction_data
gc.collect()
df_train['pickup_datetime'] = pd.to_datetime(df_train['pickup_datetime'])
 df_train['dropoff_datetime'] = pd.to_datetime(df_train['dropoff_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )<import_modules>
submission_val = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_val.loc[submission_val['sales'] < 0, 'sales'] = 0
submission_val.sort_values('id', inplace=True)
 submission_val['day'] = submission_val['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_val.columns )
<feature_engineering>
submission_eval = submission_val.copy()
submission_eval['id'] = submission_eval['id'].str.replace('validation', 'evaluation')
 submission = pd.concat([submission_val, submission_eval], axis=0, sort=False)
 print(submission.columns)
 print(submission.head(1))
df_train['month'] = df_train.pickup_datetime.dt.month
df_train['day'] = df_train.pickup_datetime.dt.dayofweek
df_train['hour'] = df_train.pickup_datetime.dt.hour
df_train['minute'] = df_train.pickup_datetime.dt.minute
df_test['month'] = df_test.pickup_datetime.dt.month
df_test['day'] = df_test.pickup_datetime.dt.dayofweek
df_test['hour'] = df_test.pickup_datetime.dt.hour
df_test['minute'] = df_test.pickup_datetime.dt.minute<feature_engineering>
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )