Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<data_type_conversions>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'], format='%Y-%m-%d %H:%M:%S' )<feature_engineering>
df= pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv")
 sell_prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv")
 calendar = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv" )
train['hour'] = train.loc[:,'pickup_datetime'].dt.hour;
train['week'] = train.loc[:,'pickup_datetime'].dt.week;
train['weekday'] = train.loc[:,'pickup_datetime'].dt.weekday;
train['hour'] = train.loc[:,'pickup_datetime'].dt.hour;
train['month'] = train.loc[:,'pickup_datetime'].dt.month;<feature_engineering>
del df,store_agg,cat_agg,agg_sales,one,x
gc.collect()
test['hour'] = test.loc[:,'pickup_datetime'].dt.hour;
test['week'] = test.loc[:,'pickup_datetime'].dt.week;
test['weekday'] = test.loc[:,'pickup_datetime'].dt.weekday;
test['hour'] = test.loc[:,'pickup_datetime'].dt.hour;
test['month'] = test.loc[:,'pickup_datetime'].dt.month;<define_variables>
warnings.filterwarnings('ignore' )
cat_vars = ['store_and_fwd_flag']<data_type_conversions>
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar = reduce_mem_usage(calendar)
 sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 sell_prices = reduce_mem_usage(sell_prices)
 sales_train_validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 submission = reduce_mem_usage(submission )
for col in cat_vars:
train[col] = train[col].astype('category' ).cat.codes
train.head()<data_type_conversions>
sales_train_validation = pd.melt(sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation = reduce_mem_usage(sales_train_validation )
for col in cat_vars:
test[col] = test[col].astype('category' ).cat.codes
test.head()<prepare_x_and_y>
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test2 = submission[submission['id'].isin(test2_rows)]
test2.columns = ['id', 'd_1942', 'd_1943', 'd_1944', 'd_1945', 'd_1946', 'd_1947', 'd_1948', 'd_1949', 'd_1950', 'd_1951', 'd_1952', 'd_1953', 'd_1954', 'd_1955', 'd_1956', 'd_1957', 'd_1958', 'd_1959',
'd_1960', 'd_1961', 'd_1962', 'd_1963', 'd_1964', 'd_1965', 'd_1966', 'd_1967', 'd_1968', 'd_1969']
product = sales_train_validation[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
test2 = test2.merge(product, how = 'left', on = 'id')
 test2 = pd.melt(test2,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation['part'] = 'train'
test2['part'] = 'test'
selection_train = ["passenger_count","store_and_fwd_flag", "pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude", "week", "weekday", "hour", "month"]
selection_test = ["passenger_count","store_and_fwd_flag", "pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude", "week", "weekday", "hour", "month"]
y_train = np.log(train["trip_duration"])
 X_train = train[selection_train]
X_test = test[selection_test] 
<normalization>
data = pd.concat([sales_train_validation,test2], axis = 0)
 del sales_train_validation,test2
gc.collect()
X = np.random.randn(10, 3)
 scaler = StandardScaler()
scaler.fit(X )<normalization>
data.reset_index(drop=True,inplace=True)
 data = data.loc[30000000:]
calendar.drop(['weekday', 'wday', 'month', 'year'], inplace = True, axis = 1)
 data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1 )
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
 X_test_scaled = scaler.transform(X_test )<import_modules>
data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 del calendar,sell_prices,product
gc.collect()
from sklearn.ensemble import RandomForestRegressor<import_modules>
nan_features = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in nan_features:
data[feature].fillna('unknown', inplace = True)
 cat = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in cat:
encoder = LabelEncoder()
data[feature] = encoder.fit_transform(data[feature] )
from sklearn.ensemble import RandomForestRegressor<train_model>
data = reduce_mem_usage(data )
mrf = RandomForestRegressor(n_estimators=20)
 mrf.fit(X_train, y_train )<import_modules>
data['lag_price_t1'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1))
data['price_change_t1'] =(data['lag_price_t1'] - data['sell_price'])/(data['lag_price_t1'])
 data['rolling_price_max_t365'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1 ).rolling(365 ).max())
 data['price_change_t365'] =(data['rolling_price_max_t365'] - data['sell_price'])/(data['rolling_price_max_t365'])
 data['rolling_price_std_t7'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(7 ).std())
 data['rolling_price_std_t30'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(30 ).std())
 data.drop(['rolling_price_max_t365', 'lag_price_t1'], inplace = True, axis = 1 )
from sklearn.model_selection import cross_val_score<feature_engineering>
data['date'] = pd.to_datetime(data['date'])
 data['year'] = data['date'].dt.year
data['month'] = data['date'].dt.month
data['week'] = data['date'].dt.week
data['day'] = data['date'].dt.day
data['dayofweek'] = data['date'].dt.dayofweek
<predict_on_test>
def weekend(arg):
if arg==5 or arg==6:
return 1
else:
return 0
data['isweekend'] = data['dayofweek'].apply(weekend )
y_test_pred = mrf.predict(X_test)
 y_test_pred[:5]<load_from_csv>
data['revenue'] = data['demand'] * data['sell_price']
data['lag_revenue_t1'] = data.groupby(['id'])['revenue'].transform(lambda x: x.shift(28))
data['rolling_revenue_std_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).std())
 data['rolling_revenue_mean_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).mean())
 data.drop(['revenue'],axis=1,inplace=True )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.head()<save_to_csv>
features = ['item_id', 'cat_id', 'state_id', 'year', 'month', 'week', 'day', 'dayofweek', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2',
'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90',
'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30', 'rolling_skew_t30', 'rolling_kurt_t30',
'isweekend','lag_revenue_t1','rolling_revenue_std_t28','rolling_revenue_mean_t28'
]
data = reduce_mem_usage(data )
submission['trip_duration']= np.exp(y_test_pred)
 submission.to_csv('asmasem_submission.csv', index=False )<load_from_csv>
x_train = data[data['part'] == 'train']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-04-24')&(data['part'] == 'train')]
y_val = x_val['demand']
test = data[data['date'] > '2016-04-24']
test.loc[test['part']=='train','id'] = test.loc[test['part']=='train','id'].str.replace('_evaluation','_validation')
 x_train.drop(['demand','part',],inplace=True,axis=1)
 x_val.drop(['demand','part',],inplace=True,axis=1)
 test.drop(['demand','part',],inplace=True,axis=1 )
train_df = pd.read_csv('.. /input/nyc-taxi-trip-duration/train.csv', nrows = 1_000_000)
 train_df.dtypes 
<prepare_x_and_y>
del data
gc.collect()
y = np.log1p(train_df['trip_duration'])
<normalization>
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'learning_rate': 0.1}
def calcul_distance(df):
pickedup =(df['pickup_latitude'], df['pickup_longitude'])
 dropoff =(df['dropoff_latitude'], df['dropoff_longitude'])
 return haversine(pickedup, dropoff )<feature_engineering>
TEST = None
for i in category:
for j in category2:
x_t,x_v,y_t,y_v,t = dataset(i,j)
 train_set = Dataset(x_t[features], y_t)
 val_set = Dataset(x_v[features], y_v)
 del x_t, y_t
gc.collect()
model = train(params, train_set, num_boost_round = 200, early_stopping_rounds = 40, valid_sets = [train_set, val_set], verbose_eval = 1000)
 y_pred = model.predict(t[features])
 t['demand'] = y_pred
TEST = pd.concat([TEST,t],axis=0)
 del x_v,y_v,t,train_set,val_set,y_pred,model
gc.collect()
train_df['distance'] = train_df.apply(lambda x : calcul_distance(x), axis = 1)
<count_values>
predictions1 = TEST[TEST['id'].apply(lambda x: "validation" in x)][['id', 'date', 'demand']]
predictions2 = TEST[TEST['id'].apply(lambda x: "evaluation" in x)][['id', 'date', 'demand']]
train_df['passenger_count'].value_counts()
train_df['vendor_id'].value_counts()<data_type_conversions>
predictions = pd.concat([prediction1,prediction2],axis=0)
 predictions.to_csv("submissionstoreanddeptwise.csv",index=False )
<data_type_conversions><EOS> 
!gzip submissionstoreanddeptwise.csv