Kaggle competition: Predict Future Sales
class AE_Cell_Net(nn.Module):
def __init__(self):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(100),
nn.utils.weight_norm(nn.Linear(100, 70)) ,
nn.ReLU6() ,
nn.BatchNorm1d(70),
nn.utils.weight_norm(nn.Linear(70, 40)) ,
nn.ReLU6() , 
)
 self.decoder = nn.Sequential(
 nn.BatchNorm1d(40),
nn.utils.weight_norm(nn.Linear(40, 70)) ,
nn.ReLU6() ,
nn.BatchNorm1d(70),
nn.utils.weight_norm(nn.Linear(70, 100)) , 
)
 def forward(self, x, mode='reconstruct'):
x = self.encoder(x)
 if mode == 'reconstruct':
x = self.decoder(x)
 return x
ae_cell_net = AE_Cell_Net().float().to(device)
 print(ae_cell_net)
 summary(ae_cell_net,(100,))<split>
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )
X_whole = torch.cat(( X[:, 1:773], X_test[:, 1:773]), 0)
 whole_ds = MoADataset(X_whole)
 whole_dl = torch.utils.data.DataLoader(whole_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 ae_gene_net, ae_gene_min_loss = train_ae('AE_Gene', whole_dl )<split>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]
X_whole = torch.cat(( X[:, 773:873], X_test[:, 773:873]), 0)
 whole_ds = MoADataset(X_whole)
 whole_dl = torch.utils.data.DataLoader(whole_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 ae_cell_net, ae_cell_min_loss = train_ae('AE_Cell', whole_dl )<define_search_space>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
X_AE_Gene = torch.zeros([X.shape[0], 309])
 X_AE_Cell = torch.zeros([X.shape[0], 40])
 X_test_AE_Gene = torch.zeros([X_test.shape[0], 309])
 X_test_AE_Cell = torch.zeros([X_test.shape[0], 40])
 with torch.no_grad() :
for i in range(0, X.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X.shape[0]:
end = X.shape[0]
else:
end = start+BATCH_SIZE
X_AE_Gene[start:end] = ae_gene_net(X[start:end, 1:773].to(device), mode='' ).cpu()
X_AE_Cell[start:end] = ae_cell_net(X[start:end, 773:873].to(device), mode='' ).cpu()
for i in range(0, X_test.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X_test.shape[0]:
end = X_test.shape[0]
else:
end = start+BATCH_SIZE
X_test_AE_Gene[start:end] = ae_gene_net(X_test[start:end, 1:773].to(device), mode='' ).cpu()
X_test_AE_Cell[start:end] = ae_cell_net(X_test[start:end, 773:873].to(device), mode='' ).cpu()
X = torch.cat(( X, X_AE_Gene, X_AE_Cell), 1)
 X_test = torch.cat(( X_test, X_test_AE_Gene, X_test_AE_Cell), 1 )<train_model>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
X_length, X_test_length = X.shape[0], X_test.shape[0]
X_temp = torch.cat(( X, X_test), 0)
 print('Shape before:', X_temp.shape)
 selector = VarianceThreshold(0.5)
 X_temp = selector.fit_transform(X_temp)
 X_temp = torch.tensor(X_temp)
 print('Shape after:', X_temp.shape)
 X = X_temp[0:X_length]
X_test = X_temp[X_length:X_length+X_test_length]<choose_model_class>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )
def train_mlp(train_dl, val_dl, seed, fold):
train_loss = 0
val_loss = 0
lowest_val_loss = np.Inf
last_epoch_with_lowest_val_loss = 0
net = MLPNet().float().to(device)
 criterion = nn.BCELoss()
optimizer = torch_optimizer.AdaBelief(net.parameters() , lr=0.1, weight_decay=0.00002, rectify=False, weight_decouple=False)
 scheduler = torch.optim.lr_scheduler.OneCycleLR(
 optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1.0, epochs=N_EPOCHS, steps_per_epoch=len(train_dl)
)
 for epoch in range(N_EPOCHS):
net.train()
for X_local, y_local in train_dl:
X_local = X_local.to(device)
 y_local = y_local.to(device)
 optimizer.zero_grad()
output = net(X_local)
 loss = criterion(output, y_local)
 loss.backward()
optimizer.step()
scheduler.step()
train_loss += loss.item()
net.eval()
for X_local, y_local in val_dl:
X_local = X_local.to(device)
 y_local = y_local.to(device)
 output = net(X_local)
 loss = criterion(output, y_local)
 val_loss += loss.item()
train_loss /= len(train_dl)
 val_loss /= len(val_dl)
 print(f'Epoch: {epoch+1:02d}/{N_EPOCHS:02d} | Train loss: {train_loss:.08f} - Val loss: {val_loss:.08f}{"(Saving model state!)" if val_loss < lowest_val_loss else ""}')
 if val_loss < lowest_val_loss:
torch.save(net.state_dict() , f'./model_mlp_{seed:02d}_{fold:02d}.state_dict')
 lowest_val_loss = val_loss
last_epoch_with_lowest_val_loss = epoch
if last_epoch_with_lowest_val_loss + 30 < epoch:
print('Early stopping!')
 break
train_loss = 0
val_loss = 0
net = MLPNet().float().to(device)
 net.load_state_dict(torch.load(f'./model_mlp_{seed:02d}_{fold:02d}.state_dict'))
return net, lowest_val_loss<normalization>
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )
class MLPNet(nn.Module):
def __init__(self):
super().__init__()
self.bn1 = nn.BatchNorm1d(X.shape[1])
 self.drop1 = nn.Dropout(0.2)
 self.fc1 = nn.utils.weight_norm(nn.Linear(X.shape[1], 1280))
self.bn2 = nn.BatchNorm1d(1280)
 self.drop2 = nn.Dropout(0.5)
 self.fc2 = nn.utils.weight_norm(nn.Linear(1280, 1280))
self.bn3 = nn.BatchNorm1d(1280)
 self.drop3 = nn.Dropout(0.5)
 self.fc3 = nn.utils.weight_norm(nn.Linear(1280, y.shape[1]))
self.relu = nn.ReLU6()
self.sigmoid = nn.Sigmoid()
def forward(self, x):
x = self.bn1(x)
 x = self.drop1(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.bn2(x)
 x = self.drop2(x)
 x = self.fc2(x)
 x = self.relu(x)
 x = self.bn3(x)
 x = self.drop3(x)
 x = self.fc3(x)
 x = self.sigmoid(x)
 return x
mlp_net = MLPNet().float().to(device)
 print(mlp_net)
 summary(mlp_net,(X.shape[1],))<train_model>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
mlp_nets = []
mlp_min_losses = []
mlp_log_losses = []
for seed in SEEDS:
set_seed(seed)
 skf = MultilabelStratifiedKFold(n_splits=N_FOLDS, random_state=seed, shuffle=True)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X, y)) :
print(f'Seed: {seed}, fold: {fold+1}/{N_FOLDS}')
 X_train, y_train = X[train_idx], y[train_idx]
train_ds = MoADataset(X_train, y_train)
 train_dl = torch.utils.data.DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 X_val, y_val = X[val_idx], y[val_idx]
val_ds = MoADataset(X_val, y_val)
 val_dl = torch.utils.data.DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)
 mlp_net, mlp_min_loss = train_mlp(train_dl, val_dl, seed, fold)
 mlp_nets.append(mlp_net)
 mlp_min_losses.append(mlp_min_loss)
 y_pred = torch.zeros(y_val.shape)
 with torch.no_grad() :
for i in range(0, X_val.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X_val.shape[0]:
end = X_val.shape[0]
else:
end = start+BATCH_SIZE
y_pred[start:end] = mlp_net(X_val[start:end].to(device)).cpu()
mlp_log_losses.append(log_loss(y_val, y_pred))
print('='*82)
 print('Training Ended! ' )<compute_test_metric>
train['revenue'] = train['item_price'] * train['item_cnt_day']
print(f'AE Gene loss: {ae_gene_min_loss}')
 print(f'AE Cell loss: {ae_cell_min_loss}')
 print('MLP CV min loss:')
 [print(l)for l in mlp_min_losses]
for i in range(len(SEEDS)) :
print(f'MLP average CV min loss(SEED {SEEDS[i]}): {sum(mlp_min_losses[i*N_FOLDS:i*N_FOLDS+N_FOLDS])/N_FOLDS}')
 print(f'MLP average CV min loss: {sum(mlp_min_losses)/len(mlp_min_losses)}')
 print('MLP log loss:')
 [print(l)for l in mlp_log_losses]
for i in range(len(SEEDS)) :
print(f'MLP average CV log loss(SEED {SEEDS[i]}): {sum(mlp_log_losses[i*N_FOLDS:i*N_FOLDS+N_FOLDS])/N_FOLDS}')
 print(f'MLP average CV log loss: {sum(mlp_log_losses)/len(mlp_log_losses)}' )<feature_engineering>
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts
with torch.no_grad() :
for i in range(0, X_test.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X_test.shape[0]:
end = X_test.shape[0]
else:
end = start+BATCH_SIZE
test_results = torch.zeros([end-start, y.shape[1]])
 for mlp_net in mlp_nets:
test_results += mlp_net(X_test[start:end].to(device)).cpu()
test_results /= int(N_FOLDS * len(SEEDS))
df_submission.iloc[start:end, 1:] = test_results<save_to_csv>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
df_submission.to_csv('submission.csv', index=False )<train_model>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
X, y, genes, cells, classnames, features, X_test, test_control, submission = load()
splits = sorted(glob.glob(".. /input/moa-pipelines/x*"))
folds, pipes = list(zip(*[joblib.load(split)for split in splits]))
print(splits )<init_hyperparams>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
device =('cuda' if torch.cuda.is_available() else 'cpu')
 pseudo = 1
pseudo_optim =(torch.optim.SGD, dict(lr=1e-5, momentum=0.9, nesterov=True))
early_stopping = -1
lr = 2e-3
wd = 1e-5
top_bias = 0.0
smoothing = 3e-4
clipping = 1e-4
batch_size = 128
dropout = [0.2, 0.2]
lookahead = 10
epochs =(30, 5)
 df = 1000
max_lr = 0.01
ps = 0.15
hidden_size =(1024, 512)
 activation = nn.ReLU
model_params = {"num_features": X.shape[1], "num_targets": len(classnames), "num_units": hidden_size,
"dropout": dropout, "top_bias": top_bias, "activation": activation}
optim_params = {"lr": lr, "weight_decay": wd}
sched_params = {"epochs": epochs[0], "pct_start": ps, "div_factor": df, "max_lr": max_lr}
fitter_params = dict(
 estimator=MLP,
estimator_params=model_params,
optim_params=optim_params,
sched_params=sched_params,
lookahead=lookahead,
pseudo=pseudo,
pseudo_optim=pseudo_optim,
savepath="validation.pth",
device=device,
loss_fn=LabelSmoothingCrossEntropy(smoothing, True, clipping)
)
 fitter = Fitter(**fitter_params)
 train_params = {"epochs": epochs, "batch_size": batch_size, "early_stopping": early_stopping, "verbose": 100}
postprocessor = None<compute_train_metric>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
%%time
seeds = np.arange(len(folds))
oof_train, oof_test = mlp_oof(X, y, X_test[~test_control], fitter, train_params, folds, seeds, pipes, postprocessor )<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
preds_clipping = 1e-5
oof_train = np.clip(oof_train, preds_clipping, 1 - preds_clipping)
 oof_test = np.clip(oof_test, preds_clipping, 1 - preds_clipping )<compute_test_metric>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
oof_result(y, oof_train, oof_test )<load_from_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
prediction = join_control_group(oof_test, test_control, submission, classnames)
 submit_preds(prediction, submission, test_control, classnames)
 joblib.dump(oof_train, "oof_train.pkl")
 pd.read_csv("submission.csv" ).iloc[:5, :5]<import_modules>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
import datetime
import random
import os
from sklearn.metrics import roc_auc_score
from sklearn.metrics import f1_score<init_hyperparams>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
config = {
'folds':15, 'lr': 0.00028, 'seed': 42,
'batch size': 512,
'inf_in':7, 'inf_out':32,
'g_in': 256, 'env2': 256, 'env3': 512,
'g_in': 772, 'g_h1': 256, 'g_h2': 256, 'g_out': 555, 'g_dropout':0.3015,
'c_in': 100, 'c_h1': 256, 'c_h2': 256, 'c_out': 543, 'c_dropout':0.3015,
'merge_out':2048, 'classes_h1': 4096, 'classes_h2': 512, 'classes':206, 'dropout':0.3015,
}<set_options>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
seed_everything(config['seed'] )<load_from_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
train_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_features.csv'))
train_targets_nonscored = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv'))
train_targets_scored = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
test_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/test_features.csv'))<define_search_space>
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
cp_type={'trt_cp':np.array([1,0]), 'ctl_vehicle':np.array([0,1])}
cp_time={24:np.array([0,0,1]), 48:np.array([0,1,0]), 72:np.array([1,0,0])}
cp_dose={'D1':np.array([1,0]), 'D2':np.array([0,1])}
class Dataset:
def __init__(self, x, y):
self.x = x
self.y = y.set_index('sig_id')
 def __getitem__(self, item):
row = self.x.iloc[item].values
y = self.y.loc[row[0]]
xg = np.float32(row[4:4+772])
 xc = np.float32(row[4+772:])
 X = np.float32(np.concatenate([cp_type[row[1]], cp_time[row[2]], cp_dose[row[3]], xg, xc]))
Y = np.float32(y.values)
 return X, Y
def __len__(self):
return len(self.x)
 class DatasetTest:
def __init__(self, x):
self.x = x
def __getitem__(self, item):
row = self.x.iloc[item].values
X = np.float32(np.concatenate([cp_type[row[1]], cp_time[row[2]], cp_dose[row[3]], row[4:]]))
id_ = row[0]
return X, id_
def __len__(self):
return len(self.x )<normalization>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1), self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def init_layer(layer):
nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, "bias"):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):
bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.0)
 class Mish(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
x = x *(torch.tanh(F.softplus(x)))
 return x
class SigmoidSoftplus(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
x = torch.sigmoid(x*F.softplus(x))
return x
class FC(nn.Module):
def __init__(self, in_features, h1, h2, out_features, drop, layer_end=True):
super().__init__()
self.l1 = nn.Linear(in_features, h1)
 self.bn1 = nn.BatchNorm1d(h1)
 self.act1 = nn.LogSigmoid()
self.dr1 = nn.Dropout(drop)
 self.l2 = nn.Linear(h1, h2)
 if layer_end == True:
self.bn2 = nn.BatchNorm1d(h2)
 self.act2 = nn.LogSigmoid()
self.dr2 = nn.Dropout(drop)
 self.l3 = nn.utils.weight_norm(nn.Linear(h2, out_features))
else:
self.l3 = None
init_layer(self.l1)
 init_layer(self.l2)
 if layer_end:
init_layer(self.l3)
 init_bn(self.bn1)
 if layer_end:
init_bn(self.bn2)
 def forward(self, x):
x = self.l1(x)
 x = self.bn1(x)
 x = self.act1(x)
 x = self.dr1(x)
 x = self.l2(x)
 if self.l3 is not None:
x = self.bn2(x)
 x = self.act2(x)
 x = self.dr2(x)
 x = self.l3(x)
 return x
class ENV(nn.Module):
def __init__(self, config):
super().__init__()
self.bn_inf = nn.BatchNorm1d(config['inf_in'])
 self.inf = nn.Linear(config['inf_in'], config['inf_out'])
 self.g = FC(config['g_in'], config['g_h1'], config['g_h2'], config['g_out'], config['g_dropout'], True)
 self.c = FC(config['c_in'], config['c_h1'], config['c_h2'], config['c_out'], config['c_dropout'], True)
 self.merge = nn.utils.weight_norm(nn.Linear(config['g_out'] + config['c_out'] + 32, config['merge_out']))
init_bn(self.bn_inf)
 init_layer(self.inf)
 def forward(self, x):
x_0 = x
x1 = x[:, :7]
x2 = x[:, 7:7+772]
x3 = x[:, 7+772:]
inf = self.inf(self.bn_inf(x1))
g = self.g(x2)
 c = self.c(x3)
 x = torch.cat([g, c, inf], 1)
 x = self.merge(x)
 return x
class MOAModel(nn.Module):
def __init__(self, config):
super().__init__()
self.env = ENV(config)
 self.act1 = SigmoidSoftplus()
self.fc = FC(config['merge_out'], config['classes_h1'], config['classes_h2'], config['classes'], config['dropout'], False)
 self.bn2 = nn.BatchNorm1d(config['classes_h2'])
 self.act2 = nn.LogSigmoid()
self.dr2 = nn.Dropout(config['dropout'])
 self.l2 = nn.utils.weight_norm(nn.Linear(config['classes_h2'], config['classes']))
init_bn(self.bn2)
 init_layer(self.l2)
 def forward(self, x):
x = self.env(x)
 x = self.act1(x)
 b = x.size(0)
 x = x.reshape(b, -1)
 x = self.fc(x)
 x = self.bn2(x)
 x = self.act2(x)
 x = self.dr2(x)
 x = self.l2(x)
 return x<filter>
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
dataset_train = train_features[scored['fold'] != config['folds'] - 1].reset_index(drop=True)
 dataset_valid = train_features[scored['fold'] == config['folds'] - 1].reset_index(drop=True)
 print(len(dataset_train), len(dataset_valid))<create_dataframe>
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
BATCH_SIZE = config["batch size"]
dts_train = Dataset(dataset_train, train_targets_scored)
 dts_valid = Dataset(dataset_valid, train_targets_scored)
 dts_test = DatasetTest(test_features)
 train_loader = torch.utils.data.DataLoader(dts_train,
batch_size=BATCH_SIZE,
shuffle=True,
sampler=None,
num_workers=4)
 valid_loader = torch.utils.data.DataLoader(dts_valid,
batch_size=BATCH_SIZE,
shuffle=False,
sampler=None,
num_workers=4)
 print(len(train_loader), len(valid_loader))<categorify>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 model = MOAModel(config)
 model = model.to(device )<choose_model_class>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
criteric1 = SmoothBCEwLogits(smoothing = 0.001)
 criteric2 = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=config['lr'])
 scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, last_epoch=-1 )<choose_model_class>
matrix['month'] = matrix['date_block_num'] % 12
class GaussianNoise(nn.Module):

 def __init__(self, sigma=0.1, is_relative_detach=True):
super().__init__()
self.sigma = sigma
self.is_relative_detach = is_relative_detach
self.noise = torch.tensor(0 ).to(device)
 def forward(self, x):
if self.training and self.sigma != 0:
scale = self.sigma * x.detach() if self.is_relative_detach else self.sigma * x
sampled_noise = self.noise.repeat(*x.size() ).float().normal_() * scale
x = x + sampled_noise
return x 
<categorify>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
def mixup_data(x, y, alpha=1.0, use_cuda=True):
if alpha > 0.:
lam = np.random.beta(alpha, alpha)
 else:
lam = 1.
 batch_size = x.size() [0]
if use_cuda:
index = torch.randperm(batch_size ).cuda()
else:
index = torch.randperm(batch_size)
 mixed_x = lam * x +(1 - lam)* x[index,:]
y_a, y_b = y, y[index]
return mixed_x, y_a, y_b, lam 
<define_variables>
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
best_loss = np.inf
epoch_start = 1
model_name_best = ''<prepare_x_and_y>
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
print(datetime.datetime.now())
 EARLY_STOPPING_STEPS = 20
EARLY_STOP = True
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
noise = GaussianNoise(0.33)
 history = []
for epoch in range(epoch_start, 15000+1):
model.train()
loss_total = 0.0
count = 0
for b_idx, xy in enumerate(train_loader):
X = torch.FloatTensor(xy[0])
 Y = torch.FloatTensor(xy[1])
 X = X.to(device)
 Y = Y.to(device)
 for param in model.parameters() :
param.grad = None
k = np.random.randint(2)
 if k == 0:
pred = model(X)
 loss = criteric1(pred, Y)
 if k == 1:
mixed_x, y_a, y_b, lam = mixup_data(X, Y, 0.4)
 pred = model(mixed_x)
 loss = lam * criteric1(pred, y_a)+(1 - lam)* criteric1(pred, y_b)
 if k == 2:
X = noise(X)
 pred = model(X)
 loss = criteric1(pred, Y)
 if k == 3:
mixed_x, y_a, y_b, lam = mixup_data(X, Y, 0.4)
 mixed_x = noise(mixed_x)
 pred = model(mixed_x)
 loss = lam * criteric1(pred, y_a)+(1 - lam)* criteric1(pred, y_b)
 loss.backward()
if b_idx % 1 == 0:
optimizer.step()
scheduler.step(epoch + b_idx / len(train_loader))
pred = torch.sigmoid(pred)
 loss_total += loss.item()
count += 1
loss_toat_mean = loss_total/count
print(f"{datetime.datetime.now() }: {epoch}.Train | loss={(loss_toat_mean):.6f}")
 model.eval()
loss_total = 0.0
count = 0
for b_idx, xy in enumerate(valid_loader):
X = torch.FloatTensor(xy[0])
 Y = torch.FloatTensor(xy[1])
 X = X.to(device)
 Y = Y.to(device)
 with torch.no_grad() :
pred = model(X)
 loss = criteric2(pred, Y)
 loss_total += loss.item()
count += 1
pred = torch.sigmoid(pred)
 loss_toat_mean = loss_total/count
print(f"{datetime.datetime.now() }: {epoch}.Valid | loss={(loss_toat_mean):.6f}")
 if best_loss >= loss_total/count :
print(f"\tSave: {best_loss} -> {(loss_total/count)}")
 best_loss = loss_total/count
if os.path.isfile(model_name_best):
os.remove(model_name_best)
 model_name_best = f'{best_loss}_model_base.pt'
torch.save(model.state_dict() , model_name_best)
 early_step = 0
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
epoch_start += 1<load_pretrained>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
msd = torch.load(f'{best_loss}_model_base.pt', map_location=device)
 model.load_state_dict(msd)
 model = model.to(device )<load_from_csv>
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts
BATCH_SIZE = 1
test_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/test_features.csv'))
test_loading = torch.utils.data.DataLoader(DatasetTest(test_features),
batch_size=BATCH_SIZE,
shuffle=False,
sampler=None,
num_workers=4 )<load_from_csv>
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts
train_targets_scored = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
train_targets_scored<load_from_csv>
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;
test_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/test_features.csv'))
sample_submission = pd.DataFrame(pd.read_csv('.. /input/lish-moa/sample_submission.csv'))<predict_on_test>
data = pd.read_pickle('data.pkl' )
print(datetime.datetime.now())
 res = []
model.eval()
for b_idx, xy in enumerate(test_loading):
X = torch.FloatTensor(xy[0])
 X = X.to(device)
 with torch.no_grad() :
pred = model(X)
 pred = torch.sigmoid(pred)
 for i in range(len(pred)) :
pr = pred[i].cpu().data.numpy()
pr = np.clip(pr,0.0005,0.999)
 res.append([xy[1][i]] + pr.tolist())
 print(len(res))<create_dataframe>
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]
res_test = pd.DataFrame(res, columns=train_targets_scored.columns)
 res_test<drop_column>
X = data[data.date_block_num <= 33].drop(['item_cnt_month'], axis=1)
 Y = data[data.date_block_num <= 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
target_cols = train_targets_scored.drop('sig_id', axis=1 ).columns.values.tolist()<save_to_csv>
del data
gc.collect() ;
sub = sample_submission.drop(columns=target_cols ).merge(res_test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<define_variables>
ts = time.time()
model = XGBRegressor(
 n_jobs=-1,
learning_rate=0.1,
max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 X_train, X_valid, y_train, y_valid = train_test_split(X, Y, test_size=0.33, random_state=7)
 eval_set = [(X_valid, y_valid)]
model.fit(
 X_train,
y_train,
eval_metric="rmse",
eval_set=eval_set,
verbose=True,
early_stopping_rounds = 2)
 time.time() - ts
KAGGLE_PATH = '.. '<set_options>
Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
} )
DataLoader,
Dataset,
random_split, 
)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 for dirname, _, filenames in os.walk(f'{KAGGLE_PATH}/input/'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
print('-Save predict')
 submission.to_csv('XGBOOST_submission.csv', index=False)
 print('output file :','XGBOOST_submission','saved' )