Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<categorify>
rcParams['figure.figsize'] = 12, 4
pd.set_option('display.max_columns',100)
 for p in [np, pd, plt, sklearn, xgboost,sns,plt,re]:
print(p.__name__, p.__version__ )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
rcParams['figure.figsize'] = 12, 4
pd.set_option('display.max_columns',100)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type=='object':
df[col] = df[col].astype('category')
 if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def display_status(message):
display(message)
 clear_output(wait=True)
 print('Available datasets:')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
start = time.time()
display_status('Load aggregated data(1/3)')
 agg_data = pd.read_csv('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/agg_data.csv')
 agg_data = reduce_mem_usage(agg_data)
 X_test = agg_data[agg_data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 X_test.fillna(0,inplace=True)
 del agg_data
gc.collect()
basepath= '.. /input/competitive-data-science-predict-future-sales/'
test = pd.read_csv(basepath+"test.csv")
 display_status('Load serialized models(2/3)--- Time elapsed:{0} secs'.format(time.time() -start))
rf = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/rf_pred_30_10.model', 'rb'))
xgb = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/xgb_pred_100_10.model', 'rb'))
meta_model = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/metamodel.model', 'rb'))
display_status('Final prediction(3/3)--- Time elapsed:{0} secs'.format(time.time() -start))
meta_test = pd.DataFrame({'ID':test.index,'pred_1':rf.predict(X_test),'pred_2':xgb.predict(X_test)})
 X_meta_test = meta_test[['pred_1','pred_2']]
submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": meta_model.predict(X_meta_test ).clip(0, 20)
 })
 submission.to_csv('meta_model_pred.csv', index=False)
 submission.to_csv('submission.csv', index=False)
 print('Final prediction submission.csv file has been generated.Thanks for your patience')
 del rf, xgb, meta_model, meta_test, X_meta_test, submission, X_test
gc.collect()
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
rcParams['figure.figsize'] = 12, 4
pd.set_option('display.max_columns',100)
 basepath= '.. /input/competitive-data-science-predict-future-sales/'
items = pd.read_csv(basepath+"items.csv")
 item_categories = pd.read_csv(basepath+"item_categories.csv")
 shops = pd.read_csv(basepath+"shops.csv")
 train = pd.read_csv(basepath+"sales_train.csv")
 test = pd.read_csv(basepath+"test.csv")
 print('All data has been loaded.' )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss 
<choose_model_class>
agg_data = pd.read_csv('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/agg_data.csv')
 cols = ['holiday_num','item_cnt_month','next_month_holiday_num','vacation_period']
agg_data[cols].groupby(["holiday_num"], as_index=False
).agg({"item_cnt_month": ["sum"] }
).plot(kind='bar',x='holiday_num',y='item_cnt_month',
title='Current Month Holidays __VS__ Sales', subplots=True, rot=90)
 agg_data[cols].groupby(["next_month_holiday_num"] , as_index=False
).agg({"item_cnt_month": ["sum"] }
).plot(kind='bar',x='next_month_holiday_num',y='item_cnt_month',
title='Anticipation of num holidays next month __VS__ Sales', subplots=True, rot=90)
 days = pd.Series(['winter','etc','spring','etc','may', 'summer','summer','summer','etc','winter','winter','winter'])
 agg_data["vacation_period"] = agg_data["month"].map(days ).astype('category')
 agg_data[cols].groupby(["vacation_period"] , as_index=False
).agg({"item_cnt_month": ["sum"] }
).plot(kind='bar',x='vacation_period',y='item_cnt_month',
title='Vacation period __VS__ Sales.May is special and deserves its own category', subplots=True, rot=90)
 del agg_data, days
gc.collect()
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim)) 
<choose_model_class>
alldata = None
alldata = train.merge(items, how='inner', on='item_id')
 alldata= alldata.merge(item_categories, how='inner', on='item_category_id')
 alldata = alldata.merge(shops,how='inner', on='shop_id')
 alldata['year'] = pd.to_datetime(alldata['date'] ).dt.year
alldata['month'] = pd.to_datetime(alldata['date'] ).dt.month
alldata['year_month'] = alldata['year'].astype(str)+ '-' + alldata['month'].astype(str)
 popular_game_oct2015 = alldata.groupby(['year_month','item_category_name','item_name'],as_index=False ).sum()
cols=['date',	'date_block_num',	'item_name', 	'item_cnt_day', 'shop_id']
top_100 = popular_game_oct2015[popular_game_oct2015['year_month']=='2015-10'].sort_values(by=['item_cnt_day'], ascending=False ).iloc[:100,:]
top_100[['item_name','item_cnt_day']].head(15 ).plot.bar(x='item_name', rot=-90)
 del alldata,popular_game_oct2015,top_100
gc.collect()
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<categorify>
num_debut = len(set(test['item_id'])- set(train['item_id']))
total_test = len(set(test['item_id']))
pctg =(num_debut/total_test)*100
print('num of debut items: {0}({1}%)out of {2} items'.format(num_debut,pctg,total_test))
<define_variables><EOS> 
gc.collect()
agg_data = pd.read_csv('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/agg_data.csv')
 X_train = agg_data[agg_data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = agg_data[agg_data.date_block_num < 33]['item_cnt_month']
X_val = agg_data[agg_data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_val = agg_data[agg_data.date_block_num == 33]['item_cnt_month']
X_test = agg_data[agg_data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 X_train.fillna(0,inplace=True)
 X_val.fillna(0,inplace=True)
 X_test.fillna(0,inplace=True)
 rf_filename = 'rf_pred_30_10.model'
rf = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/'+rf_filename, 'rb'))
del agg_data
gc.collect()
predictions = []
for tree in rf.estimators_:
predictions.append(tree.predict(X_val)[None, :])
 predictions = np.vstack(predictions)
 scores = []
acc = []
temp = [0] * len(predictions[0])
 for i in range(len(predictions)) :
temp += predictions[i]
acc.append(temp/(i+1))
scores=[]
for pred in acc:
scores.append(np.sqrt(mean_squared_error(y_val, pred)))
 scores
plt.figure(figsize=(10, 6))
plt.plot(scores, linewidth=3)
 plt.xlabel('num_trees')
 plt.ylabel('RMSE')
 del scores,predictions, rf
gc.collect()