Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<define_variables>
warnings.filterwarnings('ignore')
tokenized_path = '.. /input/xlm-r-large-tokenize-xhlulu-dataset/'<load_pretrained>
def downcast_dtypes(df):
float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int32)
 return df
x_train = np.load(tokenized_path+'x_train.npy',mmap_mode='r')
 train_toxic = np.load(tokenized_path+'df_train_toxic.npy',mmap_mode='r')
 x_valid = np.load(tokenized_path+'x_valid.npy',mmap_mode='r')
 valid_toxic = np.load(tokenized_path+'df_valid_toxic.npy',mmap_mode='r' )<create_dataframe>
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 items_categories = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 sample_submission = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv')
 test_data = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
train_dataset = ArrayDataset(x_train, train_toxic)
 valid_dataset = ArrayDataset(x_valid, valid_toxic )<set_options>
test_block = sales['date_block_num'].max() + 1
test_data['date_block_num'] = test_block
test_data = test_data.drop(columns=['ID'])
 test_data.head()
del x_train, x_valid 
<set_options>
gb = sales.groupby(index_cols, as_index=False)['item_cnt_day'].sum()
gb = gb.rename(columns={'item_cnt_day': 'target'})
 all_data = pd.merge(grid, gb, how='left', on=index_cols ).fillna(0)
 gb = sales.groupby(['shop_id', 'date_block_num'], as_index=False)['item_cnt_day'].sum()
gb = gb.rename(columns={'item_cnt_day': 'target_shop'})
 all_data = pd.merge(all_data, gb, how='left', on=['shop_id', 'date_block_num'] ).fillna(0)
 gb = sales.groupby(['item_id', 'date_block_num'], as_index=False)['item_cnt_day'].sum()
gb = gb.rename(columns={'item_cnt_day': 'target_item'})
 all_data = pd.merge(all_data, gb, how='left', on=['item_id', 'date_block_num'] ).fillna(0)
 all_data = downcast_dtypes(all_data)
 del grid, gb
gc.collect() ;
gc.collect()<compute_test_metric>
cols_to_rename = list(all_data.columns.difference(index_cols))
shift_range = [1, 2, 3, 4, 5, 12]
for month_shift in tqdm_notebook(shift_range):
train_shift = all_data[index_cols + cols_to_rename].copy()
train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift
foo = lambda x: '{}_lag_{}'.format(x, month_shift)if x in cols_to_rename else x
train_shift = train_shift.rename(columns=foo)
 all_data = pd.merge(all_data, train_shift, on=index_cols, how='left' ).fillna(0)
 del train_shift
def loss_fn(outputs, targets):
return torch.nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))<categorify>
all_data = all_data[all_data['date_block_num'] >= 12]
fit_cols = [col for col in all_data.columns if col[-1] in [str(item)for item in shift_range]]
to_drop_cols = ['target_item', 'target_shop', 'target', 'date_block_num']
to_drop_cols = list(set(list(all_data.columns)) -(set(fit_cols)|set(index_cols)))+ ['date_block_num']
item_category_mapping = items[['item_id', 'item_category_id']].drop_duplicates()
all_data = pd.merge(all_data, item_category_mapping, how='left', on='item_id')
 all_data = downcast_dtypes(all_data)
 gc.collect()
def reduce_fn(vals):
return sum(vals)/ len(vals )<train_model>
dates = all_data['date_block_num']
dates_train = dates[dates < test_block]
dates_test = dates[dates == test_block]
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d[0]
targets = d[1]
ids = ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids, 
)
 loss = loss_fn(outputs, targets)
 if bi % 50 == 0:
loss_reduced = xm.mesh_reduce('loss_reduce',loss,reduce_fn)
 xm.master_print(f'bi={bi}, loss={loss_reduced}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
model.eval()
def eval_loop_fn(data_loader, model, device):
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d[0]
targets = d[1]
ids = ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids, 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 del targets_np, outputs_np
gc.collect()
return fin_outputs, fin_targets<choose_model_class>
X_train = all_data.loc[dates < test_block].drop(to_drop_cols, axis=1)
 X_test = all_data.loc[dates == test_block].drop(to_drop_cols, axis=1)
 y_train = all_data.loc[dates < test_block, 'target'].values
y_test = all_data.loc[dates == test_block, 'target'].values
def _run() :
MAX_LEN = 192
BATCH_SIZE = 16
EPOCHS = 2
train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=0, 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=BATCH_SIZE,
sampler=valid_sampler,
drop_last=False,
num_workers=0 
)
 device = xm.xla_device()
model = mx.to(device)
 xm.master_print('done loading model')
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
xm.master_print('training on train dataset')
 lr = 0.5e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 xm.master_print(f'num_training_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 for epoch in range(EPOCHS):
gc.collect()
para_loader = pl.ParallelLoader(train_data_loader, [device])
 xm.master_print('parallel loader created...training now')
 gc.collect()
train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 del para_loader
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 gc.collect()
o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 del para_loader
gc.collect()
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 auc_reduced = xm.mesh_reduce('auc_reduce',auc,reduce_fn)
 xm.master_print(f'AUC = {auc_reduced}')
 gc.collect()
xm.master_print('training on validation set')
 lr = 1.5e-5 * xm.xrt_world_size()
num_train_steps = int(len(valid_dataset)/ BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0.1*num_train_steps,
num_training_steps=num_train_steps 
)
 xm.master_print(f'num_training_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 for epoch in range(EPOCHS):
gc.collect()
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 xm.master_print('parallel loader created...training now')
 gc.collect()
train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 del para_loader
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 gc.collect()
o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 del para_loader
gc.collect()
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 auc_reduced = xm.mesh_reduce('auc_reduce',auc,reduce_fn)
 xm.master_print(f'AUC = {auc_reduced}')
 gc.collect()
xm.save(model.state_dict() , "xlm_roberta_model.bin")
<train_model>
target_range = [0, 20]
target_range
def _mp_fn(rank, flags):
a = _run()
FLAGS={}
start_time = time.time()
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )<train_model>
lgb_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread':1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.7,
'learning_rate': 0.04,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 2**7,
'bagging_freq':1,
'verbose':0
}
model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 500)
 pred_lgb = model.predict(X_test ).clip(*target_range )
print('Time taken for training: ',time.time() -start_time )<set_options>
submission = pd.DataFrame({'ID': sample_submission.ID, 'item_cnt_month': pred_lgb})
 submission.to_csv('submission.csv', index=False )