Kaggle competition: M5 Forecasting - Accuracy
plt.rcParams['figure.figsize'] = [16, 10]
%matplotlib inline 
<load_from_csv>
sales_train_validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')
 print(sales_train_validation.shape)
 calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 print(calendar.shape)
 sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 print(sell_prices.shape )
dftrain = pd.read_csv('.. /input/train.csv')
 dftest = pd.read_csv('.. /input/test.csv')
 dftrain.head()<count_missing_values>
def convert_categorical_columns(df, columns):
for column in columns:
df[column] = df[column].astype('category')
 df[column] = df[column].cat.codes
df[column] = df[column].astype('category' )
dftrain.any().isnull()<data_type_conversions>
for i in range(1914, 1914+28):
sales_train_validation['d_'+str(i)] = 0
dftrain['pickup_datetime'] = pd.to_datetime(dftrain['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')
 dftrain['dropoff_datetime'] = pd.to_datetime(dftrain['dropoff_datetime'], format='%Y-%m-%d %H:%M:%S')
 dftest['pickup_datetime'] = pd.to_datetime(dftest['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')
 dftrain.dtypes, dftest.dtypes<feature_engineering>
convert_categorical_columns(sales_train_validation, ['dept_id', 'cat_id', 'state_id'] )
<filter>
sales_data = sales_train_validation.melt(id_vars=['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name='d',
value_name='sales' )
dftrain = dftrain[dftrain['pickup_longitude'] >= -74.03]
dftrain = dftrain[dftrain['pickup_longitude'] <= -73.75]
dftrain = dftrain[dftrain['pickup_latitude'] <= 40.85]
dftrain = dftrain[dftrain['pickup_latitude'] >= 40.63]
dftrain = dftrain[dftrain['dropoff_longitude'] <= -73.75]
dftrain = dftrain[dftrain['dropoff_longitude'] >= -74.03]
dftrain = dftrain[dftrain['dropoff_latitude'] <= 40.85]
dftrain = dftrain[dftrain['dropoff_latitude'] >= 40.63]<feature_engineering>
df = sales_train_validation.loc[:, sales_train_validation.columns.str.startswith('d_')]
sales_data['lag_7'] = df.shift(7, axis=1 ).melt() ['value']
sales_data['lag_28'] = df.shift(28, axis=1 ).melt() ['value']
del df
gc.collect()
dftrain['month'] = dftrain['pickup_datetime'].dt.month
dftrain['week'] = dftrain['pickup_datetime'].dt.week
dftrain['weekday'] = dftrain['pickup_datetime'].dt.weekday
dftrain['hours'] = dftrain['pickup_datetime'].dt.hour 
<feature_engineering>
df = sales_train_validation.loc[:, sales_train_validation.columns.str.startswith('d_')]
sales_data['lag_28_rmean_7'] = df.shift(28, axis=1 ).rolling(7, axis=1 ).mean().melt() ['value']
sales_data['lag_28_rmean_28'] = df.shift(28, axis=1 ).rolling(28, axis=1 ).mean().melt() ['value']
del df
gc.collect()
dftest['month'] = dftest['pickup_datetime'].dt.month
dftest['week'] = dftest['pickup_datetime'].dt.week
dftest['weekday'] = dftest['pickup_datetime'].dt.weekday
dftest['hours'] = dftest['pickup_datetime'].dt.hour 
<feature_engineering>
df = pd.DataFrame()
for i in range(7):
d_step7 = ['d_' + str(x)for x in range(1914+28)if(x % 7)== i]
temp = sales_train_validation.loc[:, sales_train_validation.columns.isin(d_step7)]
temp = temp.shift(28, axis=1 ).rolling(4, axis=1 ).mean()
temp['id'] = sales_train_validation['id']
temp = temp.melt(id_vars=['id'], var_name='d', value_name='lag_28_rmean_4_step_7')
 temp['lag_28_rmean_4_step_7'] = temp['lag_28_rmean_4_step_7'].astype(np.float32)
 df = df.append(temp)
 sales_data = sales_data.merge(df, how='left', on=['id', 'd'])
 del df
gc.collect()
dftrain['dist'] = np.sqrt(( dftrain['pickup_latitude']-dftrain['dropoff_latitude'])**2
+(dftrain['pickup_longitude']-dftrain['dropoff_longitude'])**2)
 dftest['dist'] = np.sqrt(( dftest['pickup_latitude']-dftest['dropoff_latitude'])**2
+(dftest['pickup_longitude']-dftest['dropoff_longitude'])**2 )<data_type_conversions>
sales_data.dropna(inplace=True )
dftrain['store_and_fwd_flag'] = dftrain['store_and_fwd_flag'].astype('category' ).cat.codes
dftest['store_and_fwd_flag'] = dftest['store_and_fwd_flag'].astype('category' ).cat.codes
dftrain.head()<count_values>
sales_data['d'] = sales_data['d'].map(lambda s: int(s[2:]))
sales_data['d'] = sales_data['d'].astype(np.int16)
 sales_data['sales'] = sales_data['sales'].astype(np.int16)
 sales_data['lag_7'] = sales_data['lag_7'].astype(np.int16)
 sales_data['lag_28'] = sales_data['lag_28'].astype(np.int16)
 sales_data['lag_28_rmean_7'] = sales_data['lag_28_rmean_7'].astype(np.float32)
 sales_data['lag_28_rmean_28'] = sales_data['lag_28_rmean_28'].astype(np.float32 )
dftrain['store_and_fwd_flag'].value_counts()<data_type_conversions>
convert_categorical_columns(calendar, ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'] )
dftrain['pickup_datetime'] = dftrain['pickup_datetime'].astype(int)
 dftrain.head()<data_type_conversions>
calendar['day'] = calendar['date'].map(lambda s: s[-2:])
 calendar['is_weekend'] = calendar['weekday'].isin(['Saturday', 'Sunday'] )
dftest['pickup_datetime'] = dftest['pickup_datetime'].astype(int)
 dftest.head()<prepare_x_and_y>
calendar.drop(['date', 'weekday'], axis=1, inplace=True )
selection = ["vendor_id", "passenger_count", "pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month", "weekday", "hours", "dist", "store_and_fwd_flag"]
X = dftrain[selection]
y = dftrain["trip_duration_log"]
X_test = dftest[selection]<import_modules>
calendar['d'] = calendar['d'].map(lambda s: int(s[2:]))
calendar['wm_yr_wk'] = calendar['wm_yr_wk'].astype(np.int16)
 calendar['wday'] = calendar['wday'].astype(np.int8)
 calendar['month'] = calendar['month'].astype(np.int8)
 calendar['year'] = calendar['year'].astype(np.int16)
 calendar['d'] = calendar['d'].astype(np.int16)
 calendar['snap_CA'] = calendar['snap_CA'].astype(np.int8)
 calendar['snap_TX'] = calendar['snap_TX'].astype(np.int8)
 calendar['snap_WI'] = calendar['snap_WI'].astype(np.int8)
 calendar['day'] = calendar['day'].astype(np.int8 )
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, cross_val_score<split>
sales_data = sales_data.merge(calendar, on='d', how='left' )
X_train, X_test, y_train, y_test = train_test_split(
 X, y, test_size=.2, random_state=42, stratify=y)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape<choose_model_class>
sell_prices['sell_price'] = sell_prices['sell_price'].astype(np.float32 )
rf = RandomForestRegressor(n_estimators=100, min_samples_leaf=4, min_samples_split=15, n_jobs=-1 )<feature_engineering>
sales_data = sales_data.merge(sell_prices, how='left', on=['store_id', 'item_id', 'wm_yr_wk'] )
<train_model>
sales_data.dropna(inplace=True )
rf.fit(X, y )<predict_on_test>
convert_categorical_columns(sales_data, ['item_id', 'store_id'] )
y_pred = rf.predict(X_test )<prepare_output>
sales_data.set_index('id', inplace=True )
my_submission = pd.DataFrame({'id': dftest.id, 'trip_duration': np.exp(y_pred)})
 my_submission.head()<save_to_csv>
del sales_train_validation, calendar, sell_prices
gc.collect()
my_submission.to_csv('submission.csv',index=False )<set_options>
MIN_D = 1400
sales_data = sales_data[sales_data['d'] > MIN_D]
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 10]
warnings.filterwarnings('ignore')
 print(os.listdir(".. /input")) 
<create_dataframe>
d_lgb_train = [i for i in range(1914-28)]
d_lgb_eval = [i for i in range(1914-28, 1914)]
d_test_1w = [i for i in range(1914, 1914+7)]
d_test_2w = [i for i in range(1914+7, 1914+7+7)]
d_test_3w = [i for i in range(1914+7+7, 1914+7+7+7)]
d_test_4w = [i for i in range(1914+7+7+7, 1914+7+7+7+7)]
beluga = pd.DataFrame({'since_start_hour': [8.0, 8.0, 20.0, 28.0, 37.0, 45.0, 85.0, 85.0, 100.0, 134.0, 134.0, 143.0, 148.0],
'Score': [ 0.400, 0.398, 0.393, 0.390, 0.383, 0.380, 0.379, 0.377, 0.376, 0.376, 0.375, 0.371, 0.368]})
 others = pd.DataFrame({'since_start_hour': [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,68,69,70,71,72,73,74,75,76,77,78,79,80,81,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148],
'BestScore': [.452,.452,.452,.452,.452,.452,.421,.421,.421,.406,.398,.398,.398,.398,.394,.394,.394,.394,.391,.391,.390,.390,.390,.390,.389,.389,.389,.389,.389,.389,.389,.385,.385,.385,.385,.385,.385,.385,.385,.385,.385,.385,.385,.383,.383,.383,.383,.383,.383,.383,.383,.383,.383,.383,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.378,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.377,.373,.373,.373]})
 with plt.xkcd() :
fig, ax = plt.subplots()
ax.plot(others.since_start_hour.values, others.BestScore.values, 'b', alpha=0.8, lw=5, label='others')
 ax.plot(beluga.since_start_hour.values, beluga.Score.values, 'k', alpha=0.8, lw=5, label='Mrig')
 ax.set_ylim(0.36, 0.4)
 ax.legend(loc=0)
 ax.set_xlabel('hours since start')
 ax.set_ylabel('RMSLE')
 ax.set_title('Race for the top')
 plt.show()<load_from_csv>
lgb_train_x = sales_data.loc[sales_data['d'].isin(d_lgb_train)].drop(['sales'], axis=1)
 lgb_train_y = sales_data.loc[sales_data['d'].isin(d_lgb_train)]['sales']
lgb_eval_x = sales_data.loc[sales_data['d'].isin(d_lgb_eval)].drop(['sales'], axis=1)
 lgb_eval_y = sales_data.loc[sales_data['d'].isin(d_lgb_eval)]['sales']
test_x_1w = sales_data.loc[sales_data['d'].isin(d_test_1w)].drop(['sales'], axis=1)
 test_x_2w = sales_data.loc[sales_data['d'].isin(d_test_2w)].drop(['sales'], axis=1)
 test_x_3w = sales_data.loc[sales_data['d'].isin(d_test_3w)].drop(['sales'], axis=1)
 test_x_4w = sales_data.loc[sales_data['d'].isin(d_test_4w)].drop(['sales'], axis=1)
 del sales_data
gc.collect()
np.random.seed(1987)
 N = 100000
t0 = dt.datetime.now()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv' )<count_unique_values>
lgb_params = {
'objective': 'poisson',
'metric': 'rmse',
'seed': 101,
'learning_rate': 0.01,
}
print('Id is unique.')if train.id.nunique() == train.shape[0] else print('oops')
 print('Train and test sets are distinct.')if len(np.intersect1d(train.id.values, test.id.values)) == 0 else print('oops')
 print('We do not need to worry about missing values.')if train.count().min() == train.shape[0] and test.count().min() == test.shape[0] else print('oops')
 print('The store_and_fwd_flag has only two values {}.'.format(str(set(train.store_and_fwd_flag.unique())| set(test.store_and_fwd_flag.unique()))))<data_type_conversions>
lgb_train = lgb.Dataset(lgb_train_x, lgb_train_y)
 lgb_eval = lgb.Dataset(lgb_eval_x, lgb_eval_y, reference=lgb_train)
 model = lgb.train(lgb_params, lgb_train, num_boost_round=10000, valid_sets=lgb_eval, early_stopping_rounds=200 )
train['pickup_datetime'] = pd.to_datetime(train.pickup_datetime)
 test['pickup_datetime'] = pd.to_datetime(test.pickup_datetime)
 train.loc[:, 'pickup_date'] = train['pickup_datetime'].dt.date
test.loc[:, 'pickup_date'] = test['pickup_datetime'].dt.date
train['dropoff_datetime'] = pd.to_datetime(train.dropoff_datetime)
 train['store_and_fwd_flag'] = 1 *(train.store_and_fwd_flag.values == 'Y')
 test['store_and_fwd_flag'] = 1 *(test.store_and_fwd_flag.values == 'Y')
 train['check_trip_duration'] =(train['dropoff_datetime'] - train['pickup_datetime'] ).map(lambda x: x.total_seconds())
 duration_difference = train[np.abs(train['check_trip_duration'].values - train['trip_duration'].values)> 1]
print('Trip_duration and datetimes are ok.')if len(duration_difference[['pickup_datetime', 'dropoff_datetime', 'trip_duration', 'check_trip_duration']])== 0 else print('Ooops.' )<categorify>
model.save_model('model.txt' )
coords = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
train[['dropoff_latitude', 'dropoff_longitude']].values,
test[['pickup_latitude', 'pickup_longitude']].values,
test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 train['pickup_pca0'] = pca.transform(train[['pickup_latitude', 'pickup_longitude']])[:, 0]
train['pickup_pca1'] = pca.transform(train[['pickup_latitude', 'pickup_longitude']])[:, 1]
train['dropoff_pca0'] = pca.transform(train[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
train['dropoff_pca1'] = pca.transform(train[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
test['pickup_pca0'] = pca.transform(test[['pickup_latitude', 'pickup_longitude']])[:, 0]
test['pickup_pca1'] = pca.transform(test[['pickup_latitude', 'pickup_longitude']])[:, 1]
test['dropoff_pca0'] = pca.transform(test[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
test['dropoff_pca1'] = pca.transform(test[['dropoff_latitude', 'dropoff_longitude']])[:, 1]<compute_test_metric>
pred_y_1w = model.predict(test_x_1w, num_iteration=model.best_iteration)
 result_1w = pd.DataFrame({'d': test_x_1w['d'], 'sales': pred_y_1w})
 test_x_2w['lag_7'] = pred_y_1w
pred_y_2w = model.predict(test_x_2w, num_iteration=model.best_iteration)
 result_2w = pd.DataFrame({'d': test_x_2w['d'], 'sales': pred_y_2w})
 test_x_3w['lag_7'] = pred_y_2w
pred_y_3w = model.predict(test_x_3w, num_iteration=model.best_iteration)
 result_3w = pd.DataFrame({'d': test_x_3w['d'], 'sales': pred_y_3w})
 test_x_4w['lag_7'] = pred_y_3w
pred_y_4w = model.predict(test_x_4w, num_iteration=model.best_iteration)
 result_4w = pd.DataFrame({'d': test_x_4w['d'], 'sales': pred_y_4w} )
def haversine_array(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
def dummy_manhattan_distance(lat1, lng1, lat2, lng2):
a = haversine_array(lat1, lng1, lat1, lng2)
 b = haversine_array(lat1, lng1, lat2, lng1)
 return a + b
def bearing_array(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
train.loc[:, 'distance_haversine'] = haversine_array(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 train.loc[:, 'distance_dummy_manhattan'] = dummy_manhattan_distance(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 train.loc[:, 'direction'] = bearing_array(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 train.loc[:, 'pca_manhattan'] = np.abs(train['dropoff_pca1'] - train['pickup_pca1'])+ np.abs(train['dropoff_pca0'] - train['pickup_pca0'])
 test.loc[:, 'distance_haversine'] = haversine_array(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values)
 test.loc[:, 'distance_dummy_manhattan'] = dummy_manhattan_distance(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values)
 test.loc[:, 'direction'] = bearing_array(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values)
 test.loc[:, 'pca_manhattan'] = np.abs(test['dropoff_pca1'] - test['pickup_pca1'])+ np.abs(test['dropoff_pca0'] - test['pickup_pca0'])
 train.loc[:, 'center_latitude'] =(train['pickup_latitude'].values + train['dropoff_latitude'].values)/ 2
train.loc[:, 'center_longitude'] =(train['pickup_longitude'].values + train['dropoff_longitude'].values)/ 2
test.loc[:, 'center_latitude'] =(test['pickup_latitude'].values + test['dropoff_latitude'].values)/ 2
test.loc[:, 'center_longitude'] =(test['pickup_longitude'].values + test['dropoff_longitude'].values)/ 2<feature_engineering>
result = pd.concat([result_1w, result_2w, result_3w, result_4w])
 result['d'] = result['d'].map(lambda x: 'F' + str(x - 1913))
result = result.reset_index().pivot(index='id', columns='d', values='sales')
 result
<train_model><EOS> 
result_e = pd.DataFrame(0, index=result.index, columns=result.columns)
 result_e.index = pd.Series(result_e.index ).apply(lambda s: s.replace('validation', 'evaluation'))
pd.concat([result, result_e] ).to_csv('submission.csv' )