Kaggle competition: Kharagpur Data Analytics Group
LAGS = [7, 28]
WINDOWS = [7, 28]
FIRST = 1942
LENGTH = 28
def demand_features(df):

 for lag in LAGS:
df[f'lag_t{lag}'] = df.groupby('id')['demand'].transform(lambda x: x.shift(lag)).astype("float32")
 for w in WINDOWS:
df[f'rolling_mean_lag{lag}_w{w}'] = df.groupby('id')[f'lag_t{lag}'].transform(lambda x: x.rolling(w ).mean() ).astype("float32")
 return df
def demand_features_eval(df):

 out = df.groupby('id', sort=False ).last()
for lag in LAGS:
out[f'lag_t{lag}'] = df.groupby('id', sort=False)['demand'].nth(-lag-1 ).astype("float32")
 for w in WINDOWS:
out[f'rolling_mean_lag{lag}_w{w}'] = df.groupby('id', sort=False)['demand'].nth(list(range(-lag-w, -lag)) ).groupby('id', sort=False ).mean().astype("float32")
 return out.reset_index()
def prep_data(df, drop_d=1000):

 df = df.drop(["d_" + str(i+1)for i in range(drop_d)], axis=1)
 df = df.assign(id=df.id.str.replace("_evaluation", ""))
df = df.reindex(columns=df.columns.tolist() + ["d_" + str(FIRST + i)for i in range(LENGTH)])
 df = df.melt(id_vars=["id", "item_id", "store_id", "state_id", "dept_id", "cat_id"], var_name='d', value_name='demand')
 df = df.assign(d=df.d.str[2:].astype("int64"),
demand=df.demand.astype("float32"))
df = demand_features(df)
 df = df[df.d >(drop_d + max(LAGS)+ max(WINDOWS)) ]
df = df.merge(calendar, how="left", on="d")
 df = df.merge(selling_prices, how="left", on=["store_id", "item_id", "wm_yr_wk"])
 df = df.drop(["wm_yr_wk"], axis=1)
 for v in ["item_id", "store_id", "state_id", "dept_id", "cat_id"]:
df[v] = OrdinalEncoder(dtype="int" ).fit_transform(df[[v]] ).astype("int16")+ 1
x = list(set(df.columns)- {'id', 'd', 'demand'})
 test = df[df.d >= FIRST - max(LAGS)- max(WINDOWS)- 28]
df = df[df.d < FIRST]
xtrain, xvalid, ytrain, yvalid = train_test_split(df[x], df["demand"], test_size=0.1, shuffle=True, random_state=54)
 train = lgb.Dataset(xtrain, label = ytrain)
 valid = lgb.Dataset(xvalid, label = yvalid)
 return train, valid, test, x
def fit_model(train, valid):

 params = {
'metric': 'rmse',
'objective': 'poisson',
'seed': 200,
'force_row_wise' : True,
'learning_rate' : 0.08,
'lambda': 0.1,
'num_leaves': 63,
'sub_row' : 0.7,
'bagging_freq' : 1,
'colsample_bytree': 0.7
}
fit = lgb.train(params,
train,
num_boost_round = 2000,
valid_sets = [valid],
early_stopping_rounds = 200,
verbose_eval = 100)
 lgb.plot_importance(fit, importance_type="gain", precision=0, height=0.5, figsize=(6, 10)) ;
return fit
def pred_all(fit, test, x):

 for i, day in enumerate(np.arange(FIRST, FIRST + LENGTH)) :
test_day = demand_features_eval(test[(test.d <= day)&(test.d >= day - max(LAGS)- max(WINDOWS)) ])
 test.loc[test.d == day, "demand"] = fit.predict(test_day[x])
 return test
def pred_to_csv(test, cols=sample_submission.columns, file="submission.csv"):

 test = test.assign(id=test.id + "_" + np.where(test.d < FIRST, "validation", "evaluation"),
F="F" +(test.d - FIRST + LENGTH + 1 - LENGTH *(test.d >= FIRST)).astype("str"))
submission = test.pivot(index="id", columns="F", values="demand" ).reset_index() [cols].fillna(1)
 submission.to_csv(file, index=False)
 return True<save_to_csv>
train = pd.read_csv('.. /input/train_final.csv')
 test = pd.read_csv('.. /input/test_final.csv' )
train, valid, test, x = prep_data(sales, 1000 - 28)
 fit = fit_model(train, valid)
 pred = pred_all(fit, test, x)
 pred_to_csv(pred, cols=sample_submission.columns, file="submission.csv" )<load_from_csv>
ids = test.PRT_ID
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 def read_data() :
data = pd.read_pickle('/kaggle/input/m5-small-fe/data_small_fe.pkl')
 calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 sales_train_validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')
 train_fold_df = sales_train_validation.iloc[:,:-28]
valid_fold_df = sales_train_validation.iloc[:,-28:]
del sales_train_validation
return data, calendar, sell_prices, submission, train_fold_df, valid_fold_df<prepare_x_and_y>
del train['PRT_ID']
del test['PRT_ID']
def run_lgb(data, features, custom_asymmetric_train, custom_asymmetric_valid):
x_train = data[data['date'] <= '2016-03-27']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-03-27')&(data['date'] <= '2016-04-24')]
y_val = x_val['demand']
test = data[(data['date'] > '2016-04-24')]
del data
gc.collect()
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'n_jobs': -1,
'seed': 42,
'learning_rate': 0.1,
'bagging_fraction': 0.75,
'bagging_freq': 10,
'colsample_bytree': 0.75}
train_set = lgb.Dataset(x_train[features], y_train)
 val_set = lgb.Dataset(x_val[features], y_val)
 del x_train, y_train
model = lgb.train(params, train_set, num_boost_round = 2500, early_stopping_rounds = 50,
valid_sets = [train_set, val_set], verbose_eval = 100, fobj = custom_asymmetric_train,
feval = custom_asymmetric_valid)
 val_pred = model.predict(x_val[features])
 y_pred = model.predict(test[features])
 x_val['demand'] = val_pred
test['demand'] = y_pred
x_val = x_val[['id', 'date', 'demand']]
test = test[['id', 'date', 'demand']]
return x_val, test<create_dataframe>
from sklearn.preprocessing import LabelEncoder
class WRMSSEEvaluator(object):
group_ids =('all_id', 'state_id', 'store_id', 'cat_id', 'dept_id', 'item_id',
['state_id', 'cat_id'], ['state_id', 'dept_id'], ['store_id', 'cat_id'],
['store_id', 'dept_id'], ['item_id', 'state_id'], ['item_id', 'store_id'])
 def __init__(self,
train_df: pd.DataFrame,
valid_df: pd.DataFrame,
calendar: pd.DataFrame,
prices: pd.DataFrame):

 self.calendar = calendar.copy()
self.prices = prices.copy()
self.train_df = train_df.copy()
self.valid_df = valid_df.copy()
self.train_target_columns = [i for i in self.train_df.columns if i.startswith('d_')]
self.weight_columns = self.train_df.iloc[:, -28:].columns.tolist()
self.train_df['all_id'] = "all"
self.id_columns = [i for i in self.train_df.columns if not i.startswith('d_')]
self.valid_target_columns = [i for i in self.valid_df.columns if i.startswith('d_')]
if not all([c in self.valid_df.columns for c in self.id_columns]):
self.valid_df = pd.concat([self.train_df[self.id_columns], self.valid_df],
axis=1,
sort=False)
 self.train_series = self.trans_30490_to_42840(self.train_df,
self.train_target_columns,
self.group_ids)
 self.valid_series = self.trans_30490_to_42840(self.valid_df,
self.valid_target_columns,
self.group_ids)
 self.weights = self.get_weight_df()
self.scale = self.get_scale()
self.train_series = None
self.train_df = None
self.prices = None
self.calendar = None
def get_scale(self):

 scales = []
for i in tqdm(range(len(self.train_series))):
series = self.train_series.iloc[i].values
series = series[np.argmax(series!=0):]
scale =(( series[1:] - series[:-1])** 2 ).mean()
scales.append(scale)
 return np.array(scales)
 def get_name(self, i):

 if type(i)== str or type(i)== int:
return str(i)
 else:
return "--".join(i)
 def get_weight_df(self)-> pd.DataFrame:

 day_to_week = self.calendar.set_index("d")["wm_yr_wk"].to_dict()
weight_df = self.train_df[["item_id", "store_id"] + self.weight_columns].set_index(
 ["item_id", "store_id"] 
)
 weight_df =(
 weight_df.stack().reset_index().rename(columns={"level_2": "d", 0: "value"})
)
 weight_df["wm_yr_wk"] = weight_df["d"].map(day_to_week)
 weight_df = weight_df.merge(
 self.prices, how="left", on=["item_id", "store_id", "wm_yr_wk"] 
)
 weight_df["value"] = weight_df["value"] * weight_df["sell_price"]
weight_df = weight_df.set_index(["item_id", "store_id", "d"] ).unstack(level=2)[
"value"
]
weight_df = weight_df.loc[
zip(self.train_df.item_id, self.train_df.store_id), :
].reset_index(drop=True)
 weight_df = pd.concat(
 [self.train_df[self.id_columns], weight_df], axis=1, sort=False 
)
 weights_map = {}
for i, group_id in enumerate(tqdm(self.group_ids, leave=False)) :
lv_weight = weight_df.groupby(group_id)[self.weight_columns].sum().sum(axis=1)
 lv_weight = lv_weight / lv_weight.sum()
for i in range(len(lv_weight)) :
weights_map[self.get_name(lv_weight.index[i])] = np.array(
 [lv_weight.iloc[i]] 
)
 weights = pd.DataFrame(weights_map ).T / len(self.group_ids)
 return weights
def trans_30490_to_42840(self, df, cols, group_ids, dis=False):

 series_map = {}
for i, group_id in enumerate(tqdm(self.group_ids, leave=False, disable=dis)) :
tr = df.groupby(group_id)[cols].sum()
for i in range(len(tr)) :
series_map[self.get_name(tr.index[i])] = tr.iloc[i].values
return pd.DataFrame(series_map ).T
def get_rmsse(self, valid_preds)-> pd.Series:

 score =(( self.valid_series - valid_preds)** 2 ).mean(axis=1)
 rmsse =(score / self.scale ).map(np.sqrt)
 return rmsse
def score(self, valid_preds: Union[pd.DataFrame, np.ndarray])-> float:
assert self.valid_df[self.valid_target_columns].shape == valid_preds.shape
if isinstance(valid_preds, np.ndarray):
valid_preds = pd.DataFrame(valid_preds, columns=self.valid_target_columns)
 valid_preds = pd.concat([self.valid_df[self.id_columns], valid_preds],
axis=1,
sort=False)
 valid_preds = self.trans_30490_to_42840(valid_preds,
self.valid_target_columns,
self.group_ids,
True)
 self.rmsse = self.get_rmsse(valid_preds)
 self.contributors = pd.concat([self.weights, self.rmsse],
axis=1,
sort=False ).prod(axis=1)
 return np.sum(self.contributors )<predict_on_test>
le = LabelEncoder()
def evaluate(x_val, train_fold_df, valid_fold_df, calendar, sell_prices):
x_val = pd.pivot(x_val, index = 'id', columns = 'date', values = 'demand' ).reset_index()
x_val.columns = ['id'] + ['d_' + str(i)for i in range(1886, 1914)]
x_val = train_fold_df[['id']].merge(x_val, on = 'id')
 x_val.drop('id', axis = 1, inplace = True)
 evaluator = WRMSSEEvaluator(train_fold_df, valid_fold_df, calendar, sell_prices)
 score = evaluator.score(x_val)
 return score
def predict(test, submission):
predictions = test[['id', 'date', 'demand']]
predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand' ).reset_index()
predictions.columns = ['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation_rows = [row for row in submission['id'] if 'evaluation' in row]
evaluation = submission[submission['id'].isin(evaluation_rows)]
validation = submission[['id']].merge(predictions, on = 'id')
 final = pd.concat([validation, evaluation])
 final.to_csv('submission.csv', index = False )<train_model>
cat_cols = [f for f in train.columns if(train[f].dtype == 'object')]
for col in tqdm(cat_cols):
lbl = LabelEncoder()
lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))
 train[col] = lbl.transform(list(train[col].values.astype('str')))
 test[col] = lbl.transform(list(test[col].values.astype('str')) )
def custom_asymmetric_train(y_pred, y_true):
y_true = y_true.get_label()
residual =(y_true - y_pred ).astype("float")
 grad = np.where(residual < 0, -2 * residual, -2 * residual * 1.15)
 hess = np.where(residual < 0, 2, 2 * 1.15)
 return grad, hess
def custom_asymmetric_valid(y_pred, y_true):
y_true = y_true.get_label()
residual =(y_true - y_pred ).astype("float")
 loss = np.where(residual < 0,(residual ** 2),(residual ** 2)* 1.15)
 return "custom_asymmetric_eval", np.mean(loss), False<define_variables>
x = train.drop('SALES_PRICE',axis=1)
 y=train.SALES_PRICE
seed_everything(42)
 data, calendar, sell_prices, submission, train_fold_df, valid_fold_df = read_data()
features = [col for col in data.columns if col not in ['id', 'demand', 'part', 'date', 'wm_yr_wk', 'mean_demand_month', 'std_demand_month', 'max_demand_month', 'mean_demand_week', 'std_demand_week',
'max_demand_week']]
print(f'We are training with {len(features)} fetures')
 data.tail()<compute_train_metric>
reg = LinearRegression()
x.fillna(0,inplace=True)
 reg.fit(x,y)
x_val, test = run_lgb(data, features, custom_asymmetric_train, custom_asymmetric_valid)
 score = evaluate(x_val, train_fold_df, valid_fold_df, calendar, sell_prices)
 print(f'Our wrmsse is {score}')
 predict(test, submission )<set_options>
price = reg.predict(test.fillna(0))
sub = pd.DataFrame(ids)
 sub['SALES_PRICE'] = price
sub.to_csv('sample1.csv',index=None )
<load_from_csv><EOS> 
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 df = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD'))
create_download_link(sub )