Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<filter>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
pd.set_option('display.max_rows', 1100)
 pd.set_option('display.max_columns', 100 )
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )<define_variables>
DATA_FOLDER = '/kaggle/input/competitive-data-science-predict-future-sales/'
train = pd.read_csv(os.path.join(DATA_FOLDER, 'sales_train.csv'))
items = pd.read_csv(os.path.join(DATA_FOLDER, 'items.csv'))
item_categories = pd.read_csv(os.path.join(DATA_FOLDER, 'item_categories.csv'))
shops = pd.read_csv(os.path.join(DATA_FOLDER, 'shops.csv'))
sample_sub = pd.read_csv(os.path.join(DATA_FOLDER, 'sample_submission.csv'))
test = pd.read_csv(os.path.join(DATA_FOLDER, 'test.csv'))
cities = pd.read_csv('.. /input/shops-en/shops-en.csv' )
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]<concatenate>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1200]
median = np.median(train[train['item_id']==2973].item_price)
 train.loc[train['item_price']<0,'item_price'] = median
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]<normalization>
cities['city'] = cities['shop_name'].str.split(" " ).map(lambda x: x[1])
 cities.iloc[20,2] = "Moscow"
cities.iloc[42,2] = "St.Petersburg"
cities.iloc[43,2] = "St.Petersburg"
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass<categorify>
def optimize_memory(df):
start_mem = df.memory_usage().sum() / 1024**2
integers = ['int8','int16','int32','int64']
floats = ['float32','float64']
int_cols = [c for c in df if df[c].dtype in integers]
float_cols = [c for c in df if df[c].dtype in floats]
for i in int_cols:
df[i] = pd.to_numeric(df[i], downcast='integer')
 for i in float_cols:
df[i] = pd.to_numeric(df[i], downcast='float')
 end_mem = df.memory_usage().sum() / 1024**2
print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )<feature_engineering>
test['date_block_num'] = 34
df_sales = pd.concat([df_sales,test], ignore_index = True)
 df_sales.drop('ID',axis = 1, inplace = True)
 df_sales.fillna(0,inplace = True )
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
relation_cols = []
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1)
 relation_cols.append("g_" + stats)
 relation_cols.append("c_" + stats)
 relation_cols.append("gc_" + stats )<drop_column>
df_sales = pd.merge(df_sales,items,how = 'left',on = 'item_id' ).drop('item_name', axis = 1)
 df_sales = pd.merge(df_sales,cities,how = 'left',on = 'shop_id' ).drop('shop_name', axis = 1)
 df_sales['city_code'] = LabelEncoder().fit_transform(df_sales['city'])
 df_sales.drop('city',axis = 1,inplace = True )
categorical_cols = [cc for cc in data_all.columns if "cp_" in cc]
categorical_cols.remove("cp_type")
 categorical_cols<choose_model_class>
df_sales['month'] =df_sales['date_block_num'] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 df_sales['days'] = df_sales['month'].map(days)
 holidays = pd.Series([8,1,1,0,2,1,0,0,0,0,1,0])
 df_sales['holidays'] = df_sales['month'].map(holidays )
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = joblib.load(".. /input/moa-pca-joblib-myjob/genes.joblib")
 pca_cells = joblib.load(".. /input/moa-pca-joblib-myjob/cells.joblib")
 g_pca = [f"pca_g-{i}" for i in range(pca_genes.n_components_)]
c_pca = [f"pca_c-{i}" for i in range(pca_cells.n_components_)]
pca_genes_df = pd.DataFrame(pca_genes.transform(data_all[GENES]), columns = g_pca)
 pca_cells_df = pd.DataFrame(pca_cells.transform(data_all[CELLS]), columns = c_pca)
 pca_cols = g_pca + c_pca
data_all = pd.concat([pca_genes_df, pca_cells_df, data_all[cols_numeric + categorical_cols + relation_cols]], axis = 1)
 else:
pass<normalization>
def shift_feature(df, lags, feature,drop = True):
for step in lags:
new_col_name = feature+'_'+str(step)
 df[new_col_name] = df.groupby(['shop_id','item_id'])[feature].shift(periods = step)
 if(drop):
df.drop(feature,axis = 1,inplace = True)
 return df
def add_feature(df,grp_cols,feature):
new_df = df.groupby(grp_cols ).agg({'item_cnt_month': ['mean']})
 new_df.columns = [feature]
new_df.reset_index(inplace = True)
 df = pd.merge(df,new_df,on = grp_cols,how='left')
 return(df )
GENES_PCA = [g for g in data_all.columns if "pca_g-" in g]
CELLS_PCA = [c for c in data_all.columns if "pca_c-" in c]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["pca_g-" + stats] = getattr(data_all[GENES_PCA], stats )(axis = 1)
 data_all["pca_c-" + stats] = getattr(data_all[CELLS_PCA], stats )(axis = 1)
 data_all["pca_gc-" + stats] = getattr(data_all[GENES_PCA + CELLS_PCA], stats )(axis = 1 )<load_pretrained>
df_sales = shift_feature(df_sales,[1,2,3],'item_cnt_month',False)
 df_sales = add_feature(df_sales,['date_block_num', 'item_id'],'date_item_avg_cnt')
 df_sales = shift_feature(df_sales,[1,2,3],'date_item_avg_cnt')
 df_sales = add_feature(df_sales,['date_block_num', 'shop_id'],'date_shop_avg_cnt')
 df_sales = shift_feature(df_sales,[1,2,3],'date_shop_avg_cnt')
 df_sales = add_feature(df_sales,['date_block_num', 'item_category_id'],'date_cat_avg_cnt')
 df_sales = shift_feature(df_sales,[1,2,3],'date_cat_avg_cnt')
 df_sales = add_feature(df_sales,['date_block_num'],'date_avg_item_cnt')
 df_sales = shift_feature(df_sales,[1,2,3],'date_avg_item_cnt')
 df_sales.fillna(0,inplace = True)
 df_sales = optimize_memory(df_sales)
 gc.collect()
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )<load_pretrained>
df_sales = add_feature(df_sales,['month'],'month_target')
 df_sales = shift_feature(df_sales,[1,2,3,12],'month_target')
 df_sales = add_feature(df_sales,['month','item_id'],'month_item_target')
 df_sales = shift_feature(df_sales,[1,2,3],'month_item_target')
 df_sales = add_feature(df_sales,['month','shop_id'],'month_shop_target')
 df_sales = shift_feature(df_sales,[1,2,3],'month_shop_target')
 df_sales = add_feature(df_sales,['month','item_category_id'],'month_cat_target')
 df_sales = shift_feature(df_sales,[1,2,3,12],'month_cat_target')
 df_sales.fillna(0,inplace = True)
 df_sales = optimize_memory(df_sales)
 gc.collect()
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )<concatenate>
df_sales['date_item_day'] = df_sales['item_cnt_month'] / df_sales['days']
df_sales = shift_feature(df_sales,[1,2,3,12],'date_item_day')
 df_sales = add_feature(df_sales,['date_block_num','item_id','city_code'],'date_item_city_target')
 df_sales = shift_feature(df_sales,[1,2,3,12],'date_item_city_target')
 gb = df_sales.groupby(['shop_id','item_id'],as_index = False ).agg({ 'avg_item_price':'mean'})
 gb.rename(columns ={'avg_item_price':'avg_pair_price'},inplace = True)
 df_sales = pd.merge(df_sales,gb,how='left',on=['shop_id','item_id'] ).fillna(0)
 df_sales['delta_item_price'] =(df_sales['avg_pair_price'] - df_sales['avg_item_price'])/df_sales['avg_item_price']
df_sales = shift_feature(df_sales,[1,2,3,6],'avg_item_price')
 df_sales = shift_feature(df_sales,[1,2,3,6],'avg_pair_price')
 df_sales = shift_feature(df_sales,[1,2,3,6],'delta_item_price')
 df_sales.fillna(0,inplace = True)
 df_sales = optimize_memory(df_sales)
 del gb
gc.collect()
try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )<filter>
start_mem = df_sales.memory_usage().sum() / 1024**2
df_sales = df_sales[df_sales['date_block_num'] > 11]
end_mem = df_sales.memory_usage().sum() / 1024**2
print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
gc.collect()
cat_idx = []
for col in categorical_cols:
cat_idx.append(data_all.columns.get_loc(col))
cat_idx<import_modules>
df_sales.to_pickle('df_sales.pkl' )
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-5)+ y_true * np.log(logits + 1e-5)
 return np.mean(-aux )<choose_model_class>
MAX_EPOCH = 300
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10,
cat_idxs = cat_idx 
)
 scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 7
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 678234, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<compute_test_metric>
X_train = df_sales[df_sales['date_block_num'] < 33].drop('item_cnt_month',axis = 1)
 y_train = df_sales[df_sales['date_block_num'] < 33].item_cnt_month
X_val = df_sales[df_sales['date_block_num'] == 33].drop('item_cnt_month',axis = 1)
 y_val = df_sales[df_sales['date_block_num'] == 33].item_cnt_month
X_test = df_sales[df_sales['date_block_num'] == 34].drop('item_cnt_month',axis = 1)
 gc.collect()
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )<feature_engineering>
ts = time.time()
xgb = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
subsample=0.8,
colsample_bytree=0.8,
eta = 0.3,
seed=42)
 xgb.fit(
 X_train,
y_train,
eval_metric="rmse",
eval_set=[(X_train, y_train),(X_val, y_val)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 for col in cols_to_zero:
submission[col] = [0.000012] * len(submission)
 submission.to_csv("submission.csv", index = None)
 submission.head()<install_modules>
pickle.dump(xgb, open("xgb.dat", "wb"))
%%capture
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
y_pred = xgb.predict(X_val ).clip(0,20)
 y_pred_tr = xgb.predict(X_train ).clip(0,20)
 rmse_tr = mean_squared_error(y_train, y_pred_tr,squared=False)
 rmse_val = mean_squared_error(y_val, y_pred,squared=False)
 print("RMSE Validation: %.5f" % rmse_val)
 print("RMSE Training: %.5f" % rmse_tr )
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN<set_options>
y_test = xgb.predict(X_test ).clip(0,20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": y_test
})
 submission.to_csv('submission_1.csv', index=False )
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<define_variables>
X_train = df_sales[df_sales['date_block_num'] < 34].drop('item_cnt_month',axis = 1)
 y_train = df_sales[df_sales['date_block_num'] < 34].item_cnt_month
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 100
ncompo_cells = 20
encoding = "dummy"<load_from_csv>
param_search = {"max_iter": [1500]}
elanet = ElasticNet()
tscv = TimeSeriesSplit(n_splits = 10 ).split(X_train)
 gsearch = GridSearchCV(elanet, cv=tscv,param_grid=param_search,scoring = 'neg_root_mean_squared_error',n_jobs = 2,verbose = 3)
 gsearch.fit(X_train,y_train)
 gc.collect()
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 drug_id = pd.read_csv(data_path + "train_drug.csv" )<filter>
dates_train = X_train['date_block_num']
dates_train_level2 = dates_train[dates_train.isin([28,29,30,31,32,33])]
pred_lr = gsearch.predict(X_test ).clip(0,20)
 pred_xgb = xgb.predict(X_test ).clip(0,20)
 X_test_level2 = np.c_[pred_lr,pred_xgb]
y_train_level2 = y_train[dates_train.isin([28,29,30,31,32,33])]
X_train_level2 = np.zeros([y_train_level2.shape[0], 2] )
if no_ctl:
print(b_, "not_ctl")
 ctl_vehicle_train = train["cp_type"] != "ctl_vehicle"
train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )<define_variables>
del gsearch,pred_lr,pred_xgb
gc.collect()
ts = time.time()
lr = ElasticNet(max_iter = 1500)
 for cur_block_num in [28,29,30,31,32,33]:
print(cur_block_num)
 X_train_part = X_train[dates_train < cur_block_num]
y_train_part = y_train[dates_train < cur_block_num]
X_test_part = X_train[dates_train == cur_block_num]
y_test_part = y_train[dates_train == cur_block_num]
print('Linear Regression Model')
 lr.fit(X_train_part,y_train_part)
 pred_lr = lr.predict(X_test_part)
 print('Fitting XGBoost Model')
 xgb.fit(X_train_part,y_train_part,eval_metric="rmse", eval_set=[(X_train_part, y_train_part),(X_test_part, y_test_part)],
verbose=True, early_stopping_rounds = 10)
 pred_xgb = xgb.predict(X_test_part)
 X_train_level2[dates_train_level2 == cur_block_num,0] = pred_lr
X_train_level2[dates_train_level2 == cur_block_num,1] = pred_xgb
time.time() - ts
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]<concatenate>
pickle.dump(X_train_level2, open("X_train_level2.dat", "wb"))
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]<normalization>
alphas_to_try = np.linspace(0, 1, 5000)
 rmse_scores = np.array([mean_squared_error(y_train_level2,np.dot(X_train_level2,[alpha,1-alpha]),squared = False)for alpha in alphas_to_try])
 best_alpha = alphas_to_try[rmse_scores.argmin() ]
rmse_train_simple_mix = rmse_scores.min()
print('Best alpha: %f; Corresponding rmse score on train: %f' %(best_alpha, rmse_train_simple_mix))
<create_dataframe><EOS> 
y_test = np.dot(X_test_level2,[best_alpha,1-best_alpha] ).clip(0,20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": y_test
})
 submission.to_csv('submission_2.csv', index=False )