Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<categorify>
%matplotlib inline
sns.set(style="darkgrid")
 warnings.filterwarnings("ignore")
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.float_format', lambda x: '%.2f' % x)
 sys.version_info
<data_type_conversions>
DATA_FOLDER = '.. /input/competitive-data-science-predict-future-sales/'
train = pd.read_csv(os.path.join(DATA_FOLDER, 'sales_train.csv'))
items = pd.read_csv(os.path.join(DATA_FOLDER, 'items.csv'))
items = items.drop('item_name', axis=1)
 test = pd.read_csv(os.path.join(DATA_FOLDER, 'test.csv')).set_index('ID')
 test['date_block_num'] = 34
train['Embarked'] = train['Embarked'].fillna(value=train['Embarked'].mode() [0])
 test['Embarked'] = test['Embarked'].fillna(value=test['Embarked'].mode() [0])
<define_variables>
def compress_columns(df,columns,keyword,search_type,datatype):
if search_type=='in':
valid_features = [x for x in columns if keyword in x]
elif search_type=='start':
valid_features = [x for x in columns if x.startswith(keyword)]
if len(valid_features):
for f in valid_features:
df[f] = df[f].round().astype(datatype)
 return df
def data_compression(df):
features = df.columns.values
if 'date_block_num' in features:
df['date_block_num'] = df['date_block_num'].astype(np.int8)
 if 'shop_id' in features:
df['shop_id'] = df['shop_id'].astype(np.int8)
 if 'item_category_id' in features:
df['item_category_id'] = df['item_category_id'].astype(np.int8)
 if 'item_id' in features:
df['item_id'] = df['item_id'].astype(np.int16)
 if 'item_price' in features:
df['item_price'] = df['item_price'].astype(np.float32)
 if 'item_id_avg_item_price' in features:
df['item_id_avg_item_price'] = df['item_id_avg_item_price'].astype(np.float32)
 df = compress_columns(df,features,'item_id_sum_item_cnt_day','in',np.int16)
 df = compress_columns(df,features,'item_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'shop_id_avg_item_price','in',np.float16)
 df = compress_columns(df,features,'shop_id_sum_item_cnt_day','in',np.int16)
 df = compress_columns(df,features,'shop_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'item_category_id_avg_item_price','in',np.float16)
 df = compress_columns(df,features,'item_category_id_sum_item_cnt_day','in',np.int32)
 df = compress_columns(df,features,'item_category_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'item_cnt_day','start',np.int16)
 return df
train_ds = train[features+target]
test_ds = test[features]<categorify>
train = data_compression(train)
 items = data_compression(items)
 test = data_compression(test )
<categorify>
train = pd.merge(train,items,on='item_id',how='left')
 test = pd.merge(test,items, on='item_id', how='left' )
<set_options>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
torch.manual_seed(7 )<choose_model_class>
train_m = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day':'sum','item_price':np.mean} ).reset_index()
train_m = pd.merge(grid,train_m,on=['date_block_num','shop_id','item_id'],how='left' ).fillna(0)
 train_m = pd.merge(train_m,items,on='item_id',how='left')
 train_m = data_compression(train_m )
class Titanicnn(nn.Module):
def __init__(self,in_size,hidden_size1,hidden_size2,hidden_size3,hidden_size4,num_classes):
super().__init__()
self.linear1 = nn.Linear(in_size, hidden_size1)
 self.linear2 = nn.Linear(hidden_size1, hidden_size2)
 self.linear3 = nn.Linear(hidden_size2,hidden_size3)
 self.linear4 = nn.Linear(hidden_size3,hidden_size4)
 self.linear5 = nn.Linear(hidden_size4,num_classes)
 self.dropout = nn.Dropout(0.2)
 def forward(self,xb):
xb = xb.view(-1,xb.size(1)).float()
out = self.linear1(xb)
 out = F.relu(out)
 out = self.linear2(out)
 out = self.dropout(out)
 out = self.linear3(out)
 out = F.relu(out)
 out = self.linear4(out)
 out = self.dropout(out)
 out = self.linear5(out)
 out = self.dropout(out)
 return out
def training_step(self,batch):
features, labels = batch
out = self(features)
 loss = F.cross_entropy(out, labels)
 return loss
def validation_step(self,batch):
features,labels = batch
out = self(features)
 loss = F.cross_entropy(out, labels)
 acc = accuracy(out,labels)
 return {'valid_loss': loss, 'valid_acc': acc}
def validation_epoch_end(self,outputs):
batch_losses = [x['valid_loss'] for x in outputs]
epoch_loss = torch.stack(batch_losses ).mean()
batch_accs = [x['valid_acc'] for x in outputs]
epoch_acc = torch.stack(batch_accs ).mean()
return {'valid_loss': epoch_loss.item() , 'valid_acc': epoch_acc.item() }
def epoch_end(self,epoch,result):
print("Epoch [{}], valid_loss: {:.4f}, valid_acc: {:.4f}".format(epoch, result['valid_loss'], result['valid_acc']))<compute_test_metric>
for type_id in ['item_id', 'shop_id', 'item_category_id']:
for column_id, aggregator, aggtype in [('item_price',np.mean,'avg'),('item_cnt_day',np.sum,'sum'),('item_cnt_day',np.mean,'avg')]:
mean_df = train.groupby([type_id,'date_block_num'] ).aggregate(aggregator ).reset_index() [[column_id,type_id,'date_block_num']]
mean_df.columns = [type_id+'_'+aggtype+'_'+column_id,type_id,'date_block_num']
train_m = pd.merge(train_m, mean_df, on=['date_block_num',type_id], how='left')
 del mean_df
gc.collect()
def accuracy(outputs, labels):
_, preds = torch.max(outputs, dim=1)
 return torch.tensor(torch.sum(preds == labels ).item() / len(preds))
return preds<define_variables>
del train
gc.collect()
input_size = train_ds.shape[1] -1
hidden_size1 = 128
hidden_size2 = 1024
hidden_size3 = 512
hidden_size4 = 128
num_classes = 2<choose_model_class>
for f in train_m.columns:
if 'item_cnt' in f:
train_m[f] = train_m[f].fillna(0)
 elif 'item_price' in f:
train_m[f] = train_m[f].fillna(train_m[f].median())
 train_m = data_compression(train_m)
 train_m.info(verbose=False )
model = Titanicnn(input_size, hidden_size1=hidden_size1, hidden_size2= hidden_size2,hidden_size3 = hidden_size3,hidden_size4 = hidden_size4,num_classes = num_classes)
 model<normalization>
lag_features = list(train_m.columns[6:])+['item_cnt_day']
lags = [1,2,3,6]
def convert_to_tensors(ds, valid_size = 0.30,train_set=True):
scaler = StandardScaler()
batch_size = ds.shape[0]
if(train_set == True):
targets_t = ds.Survived.values
features_t = ds.drop(labels = ['Survived'],axis = 1 ).values
features_t = scaler.fit_transform(features_t)
 targetsTrain = torch.from_numpy(targets_t ).type(torch.LongTensor)
 featuresTrain = torch.from_numpy(features_t)
 train_tensor = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)
 valid_size = round(len(train_tensor)*(valid_size))
train_size = len(train_tensor)- valid_size
ttrain_ds, tvalid_ds = random_split(train_tensor, [train_size, valid_size])
 print("train_size:- ",len(ttrain_ds),"test_size:- ", len(tvalid_ds))
train_loader = DataLoader(ttrain_ds, batch_size = batch_size, shuffle=True, num_workers=4, pin_memory=True)
 valid_loader = DataLoader(tvalid_ds, batch_size = batch_size*2, num_workers=4, pin_memory=True)
 return train_loader,valid_loader
else:
featuresTest = ds.values
features_test = scaler.fit_transform(featuresTest)
 featuresTensor = torch.from_numpy(features_test)
 print(len(featuresTensor))
test_loader = DataLoader(featuresTensor, batch_size*2, num_workers=4, pin_memory = True)
 return test_loader 
<feature_engineering>
for lag in lags:
train_new_df = train_m.copy()
train_new_df['date_block_num'] += lag
train_new_df = train_new_df[['date_block_num','shop_id','item_id']+lag_features]
train_new_df.columns = ['date_block_num','shop_id','item_id'] + [x+'_lag_'+str(lag)for x in lag_features]
train_m = pd.merge(train_m,train_new_df,on=['date_block_num','shop_id','item_id'],how='left')
 del train_new_df
gc.collect()
print('lag %s processed' %lag )
train_loader,valid_loader = convert_to_tensors(train_ds )<compute_test_metric>
for f in train_m.columns:
if 'item_cnt' in f:
train_m[f] = train_m[f].fillna(0)
 elif 'item_price' in f:
train_m[f] = train_m[f].fillna(train_m[f].median())
 train_m = data_compression(train_m)
 train_m.info(verbose=False )
for data, labels in train_loader:
outputs = model(data)
 print(labels.shape)
 loss = F.cross_entropy(outputs, labels)
 acc = accuracy(outputs,labels)
 print('Loss:', loss.item())
 print('Initial Acc:',float(acc),"%")
 break
print('outputs.shape : ', outputs.shape)
 print('Sample outputs :
', outputs[:5].data)
 print('Sample labels :
', labels[:5].data )<set_options>
max_clip = 30
train_m['item_cnt_day'] = train_m['item_cnt_day'].clip(0,max_clip ).astype(np.float16 )
torch.cuda.is_available()<set_options>
for lag in lags:
train_new_df = train_m.copy()
train_new_df['date_block_num'] += lag
train_new_df = train_new_df[['date_block_num','shop_id','item_id']+lag_features]
train_new_df.columns = ['date_block_num','shop_id','item_id'] + [x+'_lag_'+str(lag)for x in lag_features]
test = pd.merge(test,train_new_df,on=['date_block_num','shop_id','item_id'],how='left')
 del train_new_df
gc.collect()
print('lag %s processed' %lag )
def get_default_device() :
if torch.cuda.is_available() :
return torch.device('cuda')
 else:
return torch.device('cpu' )<categorify>
for f in test.columns:
if 'item_cnt' in f:
test[f] = test[f].fillna(0)
 elif 'item_price' in f:
test[f] = test[f].fillna(test[f].median())
 test = data_compression(test )
def to_device(data,device):
if isinstance(data,(list,tuple)) :
return [to_device(x,device)for x in data]
return data.to(device,non_blocking = True )<categorify>
cols_to_drop = lag_features[:-1] + ['item_price']
print('Columns to drop')
 print(cols_to_drop )
for xd, yd in train_loader:
print(xd.shape)
 xds = to_device(xd, device)
 print(xds.device)
 break<categorify>
train_cols = train_m.columns.values
test_cols = test.columns.values
for c in cols_to_drop:
if c in train_cols:
train_m = train_m.drop(c,axis=1)
 if c in test_cols:
test = test.drop(c,axis=1 )
class DeviceDataLoader() :
def __init__(self, ds, device):
self.ds = ds
self.device = device
def __iter__(self):
for d in self.ds:
yield to_device(d,self.device)
 def __len__(self):
return len(self.ds )<load_pretrained>
train_m['month'] = train_m['date_block_num']%12
train_m['month'] = train_m['month'].astype(np.int8)
 days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 train_m['days'] = train_m['month'].map(days ).astype(np.int8)
 test['month'] = 11
test['month'] = test['month'].astype(np.int8)
 test['days'] = 30
test['days'] = test['days'].astype(np.int8 )
train_loader = DeviceDataLoader(train_loader, device)
 valid_loader = DeviceDataLoader(valid_loader, device )<train_model>
print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 33]['item_cnt_day'])
 print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 32]['item_cnt_day'])
 print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 31]['item_cnt_day'] )
for xd, yd in valid_loader:
print('xd.device:', xd.device)
 print('yd:', yd)
 break<split>
train_m = train_m[train_m['date_block_num']>12]
def evaluate(model, valid_loader):
outputs = [model.validation_step(batch)for batch in valid_loader]
return model.validation_epoch_end(outputs )<train_model>
train_set = train_m[train_m['date_block_num']<33]
val_set = train_m[train_m['date_block_num']==33]
def fit(epochs, lr, model, train_loader, valid_loader, opt_func = torch.optim.SGD):
history = []
optimizer = opt_func(model.parameters() ,lr)
 for epoch in tqdm(range(epochs)) :
for batch in train_loader:
loss = model.training_step(batch)
 loss.backward()
optimizer.step()
optimizer.zero_grad()
result = evaluate(model,valid_loader)
 model.epoch_end(epoch, result)
 history.append(result)
 return history<train_model>
train_set.to_pickle('train.pkl')
 val_set.to_pickle('val.pkl')
 test.to_pickle('test.pkl' )
model = Titanicnn(input_size,hidden_size1=hidden_size1, hidden_size2= hidden_size2, hidden_size3= hidden_size3,hidden_size4 = hidden_size4,num_classes = num_classes)
 to_device(model, device )<compute_test_metric>
del train_m
gc.collect()
history = [evaluate(model,valid_loader)]
history<train_model>
train_x = train_set.drop(['item_cnt_day'],axis=1)
 train_y = train_set['item_cnt_day']
val_x = val_set.drop(['item_cnt_day'],axis=1)
 val_y = val_set['item_cnt_day']
features = list(train_x.columns.values )
history += fit(20, 0.2, model, train_loader, valid_loader )<train_model>
def post_processing(model,model_name,train_x,val_x,test_x,train_y,val_y,test):
train_pred = model.predict(train_x ).clip(0, 20)
 val_pred = model.predict(val_x ).clip(0, 20)
 test_pred = model.predict(test_x ).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred))
print("Val RMSE: %f" %(val_rmse))
submission = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred})
 submission.to_csv('%s_submission.csv'%model_name,index=False)
 pickle.dump(lm, open("%s_model.pickle" %model_name, "wb"))
return train_pred,val_pred,test_pred
history += fit(40, 0.001, model, train_loader, valid_loader )<train_model>
scaler = preprocessing.StandardScaler()
scaler.fit(train_x.values)
 train_x_norm = scaler.transform(train_x.values)
 val_x_norm = scaler.transform(val_x.values)
 test_norm = scaler.transform(test.values )
history += fit(100, 0.01, model, train_loader, valid_loader )<train_model>
gc.collect()
ts = time.time()
lm = linear_model.Ridge()
lm.fit(train_x_norm,train_y)
 print('Training time: %s' %(time.time() - ts))
history += fit(300, 0.01, model, train_loader, valid_loader )<train_model>
train_pred1,val_pred1,test_pred1 = post_processing(lm,'ridge',train_x_norm,val_x_norm,test_norm,train_y,val_y,test )
history += fit(30, 0.001, model, train_loader, valid_loader )<categorify>
del train_x_norm
del val_x_norm
del test_norm
gc.collect()
test_loader = convert_to_tensors(test_ds,train_set= False )<predict_on_test>
gc.collect()
ts = time.time()
xgbtrain = xgb.DMatrix(train_x.values, train_y.values)
 param = {'max_depth':8,
'subsample':1,
'min_child_weight':0.5,
'eta':0.3,
'num_round':800,
'seed':1,
'verbosity':2,
'eval_metric':'rmse'}
bst = xgb.train(param, xgbtrain)
 print('Training time: %s' %(time.time() - ts))
def predict(pred_x,model):
for pred in pred_x:
y = model(pred)
 _, y_ = torch.max(y, dim=1)
 return y_<load_pretrained>
train_pred2,val_pred2,test_pred2 = post_processing(bst,'xgboost',xgb.DMatrix(train_x.values),xgb.DMatrix(val_x.values),xgb.DMatrix(test.values),train_y,val_y,test )