Kaggle competition: M5 Forecasting - Accuracy
df_train['d_latitude'] = df_train['pickup_latitude'] - df_train['dropoff_latitude']
df_train['d_longitude'] = df_train['pickup_longitude'] - df_train['dropoff_longitude']
df_test['d_latitude'] = df_test['pickup_latitude'] - df_test['dropoff_latitude']
df_test['d_longitude'] = df_test['pickup_longitude'] - df_test['dropoff_longitude']<feature_engineering>
train = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv")
 calendar = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/calendar.csv")
 price = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/sell_prices.csv")
 sample = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/sample_submission.csv" )
df_train['distance'] = np.sqrt(np.square(df_train['d_longitude'])+ np.square(df_train['d_latitude']))
df_test['distance'] = np.sqrt(np.square(df_test['d_longitude'])+ np.square(df_test['d_latitude']))<train_model>
valid_df = pd.DataFrame(np.array(train.iloc[:,-28:])).astype("int16" )
<prepare_x_and_y>
def calendar_preprocessing(df=calendar):
int16_col = ["wm_yr_wk", "wday", "month", "year", "snap_CA", "snap_TX", "snap_WI"]
df[int16_col] = df[int16_col].astype("int16")
 df["date"] = pd.to_datetime(df["date"])
 df["mday"] = df["date"].dt.day.astype("int8")
 list_col = ["event_name_1", "event_type_1", "event_name_2", "event_type_2"]
df[list_col] = df[list_col].fillna("na")
 for i in list_col:
ce_oe = ce.OrdinalEncoder(cols=i, handle_unknown='impute')
 df = ce_oe.fit_transform(df)
 df[i] = df[i].astype("int16")
 col = ['wm_yr_wk', 'wday', 'month', 'year', 'event_name_1', 'event_type_1',
'event_name_2', 'event_type_2', 'snap_CA', 'snap_TX', 'snap_WI','mday']
for c in col:
df['shift28_{}'.format(c)] = df[c].shift(-28 ).fillna(1000 ).astype("int16")
 df.drop(col, axis=1, inplace=True)
 df.drop("weekday", axis=1, inplace=True)
 df.drop(["shift28_event_type_1", "shift28_event_name_2", "shift28_event_type_2"], axis=1, inplace=True)
 return df
def price_preprocessing(df=price):
int16_col = ["wm_yr_wk"]
df[int16_col] = df[int16_col].astype("int16")
 float16_col = ["sell_price"]
df[float16_col] = df[float16_col].astype("float16")
 col = ['wm_yr_wk', 'sell_price']
df['shift28_{}'.format(col[0])] = df[col[0]].shift(-28 ).fillna(1000 ).astype("int16")
 df['shift28_{}'.format(col[1])] = df[col[1]].shift(-28 ).fillna(1000 ).astype("float16")
 df.drop(col, axis=1, inplace=True)
 return df
def train_preprocessing(df=train):
int16_col = df.loc[:,"d_1":].columns
df[int16_col] = df[int16_col].astype("int16")
 drop_col = df.loc[:,"d_1":"d_817"]
df.drop(drop_col, axis=1, inplace=True)
 id_col = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]
df = df.melt(id_vars=id_col, var_name="d", value_name="volume")
 return df
Features = ["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","distance","month","hour","day"]
Target = "trip_duration"
X_train = df_train[Features]
y_train = df_train[Target]
lgb_train = lgb.Dataset(X_train,y_train)
 lgb_params = {'learning_rate': 0.1,
'max_depth': 25,
'num_leaves': 1000,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 1000
}<train_model>
train = train_preprocessing(df=train)
 train.dtypes
model_lgb = lgb.train(lgb_params,lgb_train,num_boost_round=500 )<feature_engineering>
master = pd.merge(train, calendar, left_on="d", right_on="d", how="left")
 master = pd.merge(master, price, left_on=["store_id", "item_id", "shift28_wm_yr_wk"],
right_on=["store_id", "item_id", "shift28_wm_yr_wk"], how='left' )
<predict_on_test>
del train, calendar, price
gc.collect()
X_prediction = df_test[Features]
prediction = np.exp(model_lgb.predict(X_prediction))
print(prediction )<create_dataframe>
def Create_train_test_dataset(df=master, random_state=10):
df.dropna(inplace=True)
 id_col = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]
for i in id_col:
ce_oe = ce.OrdinalEncoder(cols=i, handle_unknown='impute')
 df = ce_oe.fit_transform(df)
 df[i] = df[i].astype("int16")
 df.drop("d", axis=1, inplace=True)
 lags = [7, 14, 28]
for l in lags:
df["lag_{}".format(l)] = df.groupby("id")["volume"].transform(lambda x: x.shift(l)).fillna(1000 ).astype("int16")
 df["lag_d_{}".format(l)] = df.groupby("id")["volume"].transform(lambda x: x.shift(l)- x.shift(0)).fillna(1000 ).astype("int16")
 window = [7,28]
for w in window:
for l in lags:
df["rolling_{}_lag_{}".format(w,l)] = df.groupby("id")["volume"].transform(lambda x: x.rolling(w ).mean().shift(l)).fillna(1000 ).astype("float16")
 df["rolling_{}_lag_d_{}".format(w,l)] = df.groupby("id")["volume"].transform(lambda x: x.rolling(w ).mean().shift(l)- x.rolling(w ).mean().shift(0)).fillna(1000 ).astype("float16")
 list_vol_flag = []
for v in df["volume"]:
if v == 0:
flg = 0
list_vol_flag.append(flg)
 elif v == 1:
flg = 1
list_vol_flag.append(flg)
 elif v == 2:
flg = 2
list_vol_flag.append(flg)
 elif v > 2 and v <= 5:
flg = 3
list_vol_flag.append(flg)
 elif v > 5 and v <=20:
flg = 4
list_vol_flag.append(flg)
 else :
flg=5
list_vol_flag.append(flg)
 df["vol_range"] = list_vol_flag
df["vol_range"] = df["vol_range"].astype("int16")
 s_lag = [1,7]
for sl in s_lag:
df["lag_price_{}".format(sl)] = df.groupby("id")["shift28_sell_price"].transform(lambda x: x.shift(sl)- x.shift(0)).fillna(1000 ).astype("float16")
 df["target"] = df.groupby("id")["volume"].transform(lambda x:x.shift(-28)).fillna(0 ).astype("int16")
 Xmas_date = [pd.datetime(2013,12,25), pd.datetime(2014,12,25), pd.datetime(2015,12,25)]
X = df[(df["date"] <= df["date"].max() - datetime.timedelta(days=28))
&(df["date"] > df["date"].min() + datetime.timedelta(days=56)) ].set_index("date" ).drop(Xmas_date ).drop(["target"], axis=1)
 y = df[(df["date"] <= df["date"].max() - datetime.timedelta(days=28))
&(df["date"] > df["date"].min() + datetime.timedelta(days=56)) ].set_index("date" ).drop(Xmas_date)["target"]
X_test = df[df["date"] > df["date"].max() - datetime.timedelta(days=28)].set_index("date" ).drop(["target"], axis=1)
 X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=random_state)
 return X_train, X_val, y_train, y_val, X_test
My_Submission = pd.DataFrame({'id': df_test.id, 'trip_duration': prediction})
 print(My_Submission )<save_to_csv>
%%time
X_train, X_val, y_train, y_val, X_test = Create_train_test_dataset(df=master, random_state=10 )
My_Submission.to_csv('submission.csv', index=False )<set_options>
del master
gc.collect()
%matplotlib inline<define_variables>
def lgbm_val(X_train, X_val, y_train, y_val):
train = lgb.Dataset(X_train, label = y_train)
 valid = lgb.Dataset(X_val, label = y_val)
 params = {
'metric': 'rmse',
'objective': 'poisson',
'seed': 200,
'force_row_wise' : True,
'learning_rate' : 0.07,
'lambda': 0.1,
'num_leaves': 60,
'sub_row' : 0.7,
'bagging_freq' : 1,
'colsample_bytree': 0.85
}
lgbm = lgb.train(params,
train,
num_boost_round = 800,
valid_sets = [valid],
early_stopping_rounds = 50,
verbose_eval = 5)
 lgb.plot_importance(lgbm, importance_type="gain", precision=0, figsize=(6, 13)) ;
pred_train = lgbm.predict(X_train)
 pred_val = lgbm.predict(X_val)
 return pred_train, pred_val, lgbm
DATA_PATH = ".. /input"
TRAIN_FILE_PATH = os.path.join(DATA_PATH, "train.csv")
 TEST_FILE_PATH = os.path.join(DATA_PATH, "test.csv")
 SAMPLE_SUB_FILE_PATH = os.path.join(DATA_PATH, "sample_submission.csv")
 SUB_FILE_PATH = os.path.join(DATA_PATH, "submission.csv" )<load_from_csv>
%%time
pred_train, pred_val, lgbm = lgbm_val(X_train, X_val, y_train, y_val )
train = pd.read_csv(TRAIN_FILE_PATH, index_col="id", parse_dates=["pickup_datetime", "dropoff_datetime"] )<split>
print("MSE train %.3f" % mean_squared_error(y_true=y_train, y_pred=pred_train))
print("MSE validation %.3f" % mean_squared_error(y_true=y_val, y_pred=pred_val))
print("R2 score train %.3f" % r2_score(y_true=y_train, y_pred=pred_train))
print("R2 score validation %.3f" % r2_score(y_true=y_val, y_pred=pred_val))
train_set, val_set = train_test_split(train, test_size=0.2, random_state=42)
 print(len(train_set), len(val_set))
train = train_set<categorify>
del X_train, X_val, y_train, y_val, pred_train, pred_val
gc.collect()
class ClipOutliers(BaseEstimator, TransformerMixin):
def __init__(self, attribute_name, lower_limit=None, upper_limit=None):
self.attribute_name = attribute_name
self.lower_limit = lower_limit
self.upper_limit = upper_limit
def fit(self, df, y=None):
return self
def transform(self, df, y=None):
if self.attribute_name not in df.columns:
return df
elif self.lower_limit != None:
return df[df[self.attribute_name] > self.lower_limit]
elif self.upper_limit != None:
return df[df[self.attribute_name] < self.upper_limit]
else:
return df
clip_outliers_in_trip_duration = ClipOutliers("trip_duration", lower_limit=None, upper_limit=6000)
 train_trip_duration_clipped = clip_outliers_in_trip_duration.fit_transform(train)
 clip_outliers_in_passenger_count = ClipOutliers("passenger_count", lower_limit=0, upper_limit=None)
 train_clipped = clip_outliers_in_trip_duration.fit_transform(train_trip_duration_clipped)
 plt.subplots(figsize=(15, 6))
train_clipped.boxplot()<define_variables>
%%time
pred_test = lgbm.predict(X_test)
 pred = np.array(pred_test, dtype='float16')
 pred = np.reshape(pred,(30490, 28), order='F')
 pred_df = pd.DataFrame(pred)
 pred_df.head()
<filter><EOS> 
col = sample.loc[:,'F1':].columns
sample[col] = pd.concat([pred_df, pred_df] ).values
sample.to_csv("submission.csv", index=False )