Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<feature_engineering>
pd.set_option('max_columns', 50)
 plt.style.use('bmh')
 color_cycle = cycle(plt.rcParams['axes.prop_cycle'].by_key() ['color'] )
class DateAttribsAdder(BaseEstimator, TransformerMixin):
def __init__(self, date_field):
self.date_field = date_field
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X["month"] = X.loc[:,self.date_field].dt.month
X["week"] = X.loc[:,self.date_field].dt.week
X["weekday"] = X.loc[:,self.date_field].dt.weekday
X["day"] = X.loc[:,self.date_field].dt.day
X["hour"] = X.loc[:,self.date_field].dt.hour
X["minute"] = X.loc[:,self.date_field].dt.minute
X["minute_of_the_day"] = X["hour"] * 60 + X.loc[:,self.date_field].dt.minute
return X.drop([self.date_field, "dropoff_datetime", "minute"], axis=1, errors="ignore")
 date_attribs_adder = DateAttribsAdder("pickup_datetime")
 date_attribs_added = date_attribs_adder.transform(taxi)
 print(date_attribs_added.head())
<categorify>
osj = os.path.join
INPUT_DIR = '.. /input/m5-forecasting-accuracy/'
train = pd.read_csv(osj(INPUT_DIR, 'sales_train_validation.csv'))
price = pd.read_csv(osj(INPUT_DIR, 'sell_prices.csv'))
calender = pd.read_csv(osj(INPUT_DIR, 'calendar.csv'))
sample_submit = pd.read_csv(osj(INPUT_DIR, 'sample_submission.csv'))
cat_attribs = ["vendor_id", "store_and_fwd_flag"]
taxi_cat = taxi[cat_attribs]
cat_encoder = OneHotEncoder()
taxi_1hot = cat_encoder.fit_transform(taxi_cat)
 print(taxi_1hot.toarray() )<prepare_x_and_y>
calender['date'] = pd.to_datetime(calender['date'] )
%%time
class DistanceAttribsAdder(BaseEstimator, TransformerMixin):
def __init__(self, attrib_name = "distance"):
self.attrib_name = attrib_name
def fit(self, df):
return self
def transform(self, df, y=None):
R=6373.0
pickup_lat=np.radians(df['pickup_latitude'].values)
 pickup_lon=np.radians(df['pickup_longitude'].values)
 dropoff_lat=np.radians(df['dropoff_latitude'].values)
 dropoff_lon=np.radians(df['dropoff_longitude'].values)
 dlon = dropoff_lon - pickup_lon
dlat = dropoff_lat - pickup_lat
a = np.sin(dlat / 2)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
df["distance"] = R * c
df.boxplot(column="distance", return_type="axes");
plt.show()
return df
distance_attribs_adder = DistanceAttribsAdder()
taxi_distance = distance_attribs_adder.transform(taxi)
 print(taxi_distance.head())
 clip_outliers_in_distance = ClipOutliers("distance", lower_limit=None, upper_limit=400)
 taxi = clip_outliers_in_distance.fit_transform(taxi_distance )<prepare_x_and_y>
d_cols = [c for c in train.columns if 'd_' in c]
class DropFeature(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X.drop([self.attribute_names],axis=1,errors="ignore" ).values<categorify>
past_sales = train.set_index('id')[d_cols].T.merge(calender.set_index('d')['date'],
left_index=True,
right_index=True,
validate='1:1' ).set_index('date' )
class FeatureSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attribute_names].values<prepare_x_and_y>
def make_total_sales_lowess(p):
total_sales = pd.DataFrame(p.sum(axis=1), columns=['total sales'])
 total_sales_noXmas = total_sales.drop(index=[dt.datetime(2011,12,25),
dt.datetime(2012,12,25),
dt.datetime(2013,12,25),
dt.datetime(2014,12,25),
dt.datetime(2015,12,25)])
 df = total_sales_noXmas
lowess = sm.nonparametric.lowess(df['total sales'], df.index, frac=.3)
 lowess_x = list(zip(*lowess)) [0]
lowess_y = list(zip(*lowess)) [1]
f = interp1d(lowess_x, lowess_y, bounds_error=False)
 new_lowess_x = df.index
new_lowess_y = f(new_lowess_x)
 total_sales_lowess = total_sales_noXmas
total_sales_lowess['lowess'] = new_lowess_y
total_sales_lowess['total sales-lowess'] = total_sales_lowess['total sales'] - total_sales_lowess['lowess']
return total_sales_lowess
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
columns = X.columns
selected_features = [feature for feature in self.attribute_names if feature in columns ]
return X[selected_features]<prepare_x_and_y>
total_sales_lowess = make_total_sales_lowess(past_sales )
class LogTransformer(BaseEstimator, TransformerMixin):
def __init__(self, attribute_name):
self.attribute_name = attribute_name
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X[self.attribute_name] = np.log(X[self.attribute_name].values)
 return X<categorify>
submission = pd.read_csv('.. /input/m5-three-shades-of-dark-darker-magic/submission_v1.csv' )
cat_pipeline = Pipeline([ 
('selector_cat', FeatureSelector(cat_attribs)) , 
("cat_trans", OneHotEncoder()),
])
<define_variables>
teams_now = 3215
teams_before = 3186
submission.loc[submission['id'].str.contains('CA_1'), 'F1'] *= 1.009
submission.loc[submission['id'].str.contains('CA_2'), 'F1'] *= 1.02
submission.loc[submission['id'].str.contains('CA_3'), 'F1'] *= 1.009
submission.loc[submission['id'].str.contains('CA_4'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('TX_1'), 'F1'] *= 1.01
submission.loc[submission['id'].str.contains('TX_2'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('TX_3'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('WI_1'), 'F1'] *= 1.015
submission.loc[submission['id'].str.contains('WI_2'), 'F1'] *= 1.02
submission.loc[submission['id'].str.contains('WI_3'), 'F1'] *= 1.015
submission['F1'].sum()
label_pipeline = Pipeline([ 
("clip_trip_duration", ClipOutliers("trip_duration", lower_limit=None, upper_limit=6000)) , 
('trip_log_transformer', LogTransformer("trip_duration")) , 
("feature_selector", FeatureSelector("trip_duration"))
] )<concatenate>
magic = submission['F2'].sum()
japan_emergency = 119
submission.loc[submission['id'].str.contains('CA_1'), 'F2'] *= 0.994
submission.loc[submission['id'].str.contains('CA_2'), 'F2'] *= 0.998
submission.loc[submission['id'].str.contains('CA_3'), 'F2'] *= 0.994
submission.loc[submission['id'].str.contains('CA_4'), 'F2'] *= 0.996
submission.loc[submission['id'].str.contains('TX_1'), 'F2'] *= 0.995
submission.loc[submission['id'].str.contains('TX_2'), 'F2'] *= 0.996
submission.loc[submission['id'].str.contains('TX_3'), 'F2'] *= 0.996
submission.loc[submission['id'].str.contains('WI_1'), 'F2'] *= 0.998
submission.loc[submission['id'].str.contains('WI_2'), 'F2'] *= 1
submission.loc[submission['id'].str.contains('WI_3'), 'F2'] *= 0.998
submission['F2'].sum()
full_pipeline = FeatureUnion(transformer_list=[ 
("num_pipeline", num_pipeline), 
("cat_pipeline", cat_pipeline),
] )<categorify>
submission.loc[submission['id'].str.contains('CA_1'), 'F3'] *= 0.994
submission.loc[submission['id'].str.contains('CA_2'), 'F3'] *= 0.998
submission.loc[submission['id'].str.contains('CA_3'), 'F3'] *= 0.994
submission.loc[submission['id'].str.contains('CA_4'), 'F3'] *= 0.996
submission.loc[submission['id'].str.contains('TX_1'), 'F3'] *= 0.995
submission.loc[submission['id'].str.contains('TX_2'), 'F3'] *= 0.996
submission.loc[submission['id'].str.contains('TX_3'), 'F3'] *= 0.996
submission.loc[submission['id'].str.contains('WI_1'), 'F3'] *= 1
submission.loc[submission['id'].str.contains('WI_2'), 'F3'] *= 1.005
submission.loc[submission['id'].str.contains('WI_3'), 'F3'] *= 1
submission['F3'].sum()
outliers_clipped = outlier_pipeline.fit_transform(train)
 taxi_prepared = full_pipeline.fit_transform(outliers_clipped)
 taxi_labels = label_pipeline.fit_transform(outliers_clipped)
 print(taxi_prepared.shape)
 taxi_prepared<categorify>
sex_constant = 1.0073
submission.loc[submission['id'].str.contains('CA_1'), 'F4'] *= 0.98
submission.loc[submission['id'].str.contains('CA_2'), 'F4'] *= 1.02
submission.loc[submission['id'].str.contains('CA_3'), 'F4'] *= 0.98
submission.loc[submission['id'].str.contains('CA_4'), 'F4'] *= 1
submission.loc[submission['id'].str.contains('TX_1'), 'F4'] *= 0.99
submission.loc[submission['id'].str.contains('TX_2'), 'F4'] *= 1
submission.loc[submission['id'].str.contains('TX_3'), 'F4'] *= 1
submission.loc[submission['id'].str.contains('WI_1'), 'F4'] *= 1.00
submission.loc[submission['id'].str.contains('WI_2'), 'F4'] *= 1.02
submission.loc[submission['id'].str.contains('WI_3'), 'F4'] *= 1.00
submission['F4'].sum()
outliers_clipped_val = outlier_pipeline.fit_transform(val_set)
 taxi_prepared_val = full_pipeline.fit_transform(outliers_clipped_val)
 taxi_labels_val = label_pipeline.fit_transform(outliers_clipped_val )<compute_train_metric>
color_of_the_night = 995874
color_of_the_night /= 1000000
submission.loc[submission['id'].str.contains('CA_1'), 'F5'] *= 0.995
submission.loc[submission['id'].str.contains('CA_2'), 'F5'] *= 1
submission.loc[submission['id'].str.contains('CA_3'), 'F5'] *= 0.995
submission.loc[submission['id'].str.contains('CA_4'), 'F5'] *= 0.998
submission.loc[submission['id'].str.contains('TX_1'), 'F5'] *= 0.996
submission.loc[submission['id'].str.contains('TX_2'), 'F5'] *= 0.998
submission.loc[submission['id'].str.contains('TX_3'), 'F5'] *= 0.998
submission.loc[submission['id'].str.contains('WI_1'), 'F5'] *= 1
submission.loc[submission['id'].str.contains('WI_2'), 'F5'] *= 1.01
submission.loc[submission['id'].str.contains('WI_3'), 'F5'] *= 1
submission['F5'].sum()
forest_reg = RandomForestRegressor(n_estimators=50,min_samples_leaf=10, min_samples_split=15, max_features='auto', max_depth=50, bootstrap=True, n_jobs=-1)
 forest_reg.fit(taxi_prepared, taxi_labels)
 predictions = forest_reg.predict(taxi_prepared_val)
 print(predictions[:5])
 print(taxi_labels_val[:5])
 score = np.sqrt(mean_squared_error(taxi_labels_val, predictions))
print("validation RMSE: {}".format(score)) 
<load_from_csv>
injury = 1.000376
submission.loc[submission['id'].str.contains('CA_1'), 'F6'] *= 1.0002
submission.loc[submission['id'].str.contains('CA_2'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('CA_3'), 'F6'] *= 1.0002
submission.loc[submission['id'].str.contains('CA_4'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('TX_1'), 'F6'] *= 1.000
submission.loc[submission['id'].str.contains('TX_2'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('TX_3'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('WI_1'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('WI_2'), 'F6'] *= 1.01
submission.loc[submission['id'].str.contains('WI_3'), 'F6'] *= 1.001
submission['F6'].sum()
test = pd.read_csv(TEST_FILE_PATH, index_col="id", parse_dates=["pickup_datetime"] )<categorify>
np.random.seed(198505)
 correction = np.random.randint(1000000)/1000000
submission.loc[submission['id'].str.contains('CA_1'), 'F7'] *= 0.994
submission.loc[submission['id'].str.contains('CA_2'), 'F7'] *= 1
submission.loc[submission['id'].str.contains('CA_3'), 'F7'] *= 0.994
submission.loc[submission['id'].str.contains('CA_4'), 'F7'] *= 0.998
submission.loc[submission['id'].str.contains('TX_1'), 'F7'] *= 0.995635
submission.loc[submission['id'].str.contains('TX_2'), 'F7'] *= 0.998
submission.loc[submission['id'].str.contains('TX_3'), 'F7'] *= 0.998
submission.loc[submission['id'].str.contains('WI_1'), 'F7'] *= 1
submission.loc[submission['id'].str.contains('WI_2'), 'F7'] *= 1.01
submission.loc[submission['id'].str.contains('WI_3'), 'F7'] *= 1
submission['F7'].sum()
outliers_clipped_test = outlier_pipeline.fit_transform(test)
 test_prepared = full_pipeline.fit_transform(test)
 print(test_prepared.shape )<predict_on_test>
submission.loc[submission['id'].str.contains('CA_1'), 'F8'] *= 0.996
submission.loc[submission['id'].str.contains('CA_2'), 'F8'] *= 1
submission.loc[submission['id'].str.contains('CA_3'), 'F8'] *= 0.996
submission.loc[submission['id'].str.contains('CA_4'), 'F8'] *= 0.998
submission.loc[submission['id'].str.contains('TX_1'), 'F8'] *= 0.9988
submission.loc[submission['id'].str.contains('TX_2'), 'F8'] *= 1
submission.loc[submission['id'].str.contains('TX_3'), 'F8'] *= 1
submission.loc[submission['id'].str.contains('WI_1'), 'F8'] *= 1
submission.loc[submission['id'].str.contains('WI_2'), 'F8'] *= 1.01
submission.loc[submission['id'].str.contains('WI_3'), 'F8'] *= 1
submission['F8'].sum()
pred_test = np.exp(forest_reg.predict(test_prepared))<load_from_csv>
submission.loc[submission['id'].str.contains('CA_1'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('CA_2'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('CA_3'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('CA_4'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('TX_1'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('TX_2'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('TX_3'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('WI_1'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('WI_2'), 'F11'] *= 1
submission.loc[submission['id'].str.contains('WI_3'), 'F11'] *= 1
submission['F11'].sum()
submit = pd.read_csv(SAMPLE_SUB_FILE_PATH)
 submit.head()<feature_engineering>
for i in range(9,20):
if i!=11:
submission.loc[submission['id'].str.contains('CA_1'), 'F'+str(i)] *= 1
submission.loc[submission['id'].str.contains('CA_2'), 'F'+str(i)] *= 1.015
submission.loc[submission['id'].str.contains('CA_3'), 'F'+str(i)] *= 1
submission.loc[submission['id'].str.contains('CA_4'), 'F'+str(i)] *= 1.013
submission.loc[submission['id'].str.contains('TX_1'), 'F'+str(i)] *= 1
submission.loc[submission['id'].str.contains('TX_2'), 'F'+str(i)] *= 1.013
submission.loc[submission['id'].str.contains('TX_3'), 'F'+str(i)] *= 1.013
submission.loc[submission['id'].str.contains('WI_1'), 'F'+str(i)] *= 1.015
submission.loc[submission['id'].str.contains('WI_2'), 'F'+str(i)] *= 1.02
submission.loc[submission['id'].str.contains('WI_3'), 'F'+str(i)] *= 1.015
print(submission['F'+str(i)].sum() )
submit["trip_duration"] = pred_test
submit.head()<save_to_csv>
for i in range(20,29):
submission.loc[submission['id'].str.contains('CA_1'), 'F'+str(i)] *= 1.015
submission.loc[submission['id'].str.contains('CA_2'), 'F'+str(i)] *= 1.03
submission.loc[submission['id'].str.contains('CA_3'), 'F'+str(i)] *= 1.015
submission.loc[submission['id'].str.contains('CA_4'), 'F'+str(i)] *= 1.025
submission.loc[submission['id'].str.contains('TX_1'), 'F'+str(i)] *= 1.02
submission.loc[submission['id'].str.contains('TX_2'), 'F'+str(i)] *= 1.025
submission.loc[submission['id'].str.contains('TX_3'), 'F'+str(i)] *= 1.025
submission.loc[submission['id'].str.contains('WI_1'), 'F'+str(i)] *= 1.03
submission.loc[submission['id'].str.contains('WI_2'), 'F'+str(i)] *= 1.04
submission.loc[submission['id'].str.contains('WI_3'), 'F'+str(i)] *= 1.03
print(submission['F'+str(i)].sum() )
submit.to_csv("submission.csv", index=False )<set_options>
submission.to_csv('submission.csv', index=False )