Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<categorify>
!pip install /kaggle/input/tensorboard-220/tensorboard-2.2.0-py3-none-any.whl -q
!pip install /kaggle/input/pytorch-lightning/pytorch_lightning-0.9.0-py3-none-any.whl -q
def fillna_means(df, col):

 df[col] = df.groupby('item_lab')[col].transform(lambda x: x.fillna(x.mean()))
df[col] = df.groupby('item_category_id')[col].transform(lambda x: x.fillna(x.mean()))<feature_engineering>
%matplotlib inline
def pre_process_item_cat_data(df):

 cat = df.item_category_name.str.split('-', n=1, expand=True)
 df['cat1'] = cat[0].str.strip().str.lower()
df['cat2'] = cat[1].str.strip().str.lower()
df.cat1.fillna('', inplace=True)
 df.cat2.fillna('', inplace=True)
 df['cat1_lab'] = df.cat1.factorize(sort=True)[0]
df['cat2_lab'] = df.cat2.factorize(sort=True)[0]<compute_train_metric>
def set_seed(seed: int = 666)-> None:
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 set_seed()
def get_string_correlation(array):
def dist(x, y):
lev = Levenshtein.distance(x[0],y[0])
 m = np.mean([len(x[0]), len(y[0])])
 return lev / m
cor = pdist(array.reshape(-1, 1), dist)
 return squareform(cor)
 def agglomerative_clustering(distance_matrix, num_clusters=None, threshold=None):
model = AgglomerativeClustering(n_clusters=num_clusters, affinity='precomputed', linkage='average', distance_threshold=threshold)
 model.fit(distance_matrix)
 return model.labels_
def add_levenshtein_feature(df, cols):
for col in cols:
categories = df[col].unique()
corr = get_string_correlation(categories)
 labels = agglomerative_clustering(corr, threshold=0.5)
 features = pd.DataFrame(np.array([categories, labels] ).T, columns=[col, col + '_lev'])
 df = df.merge(features, how='left', on=col)
 df[col + '_lev'] = df[col + '_lev'].astype('int32')
 return df<merge>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_targets_nonscored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv' )
pre_process_item_cat_data(ITEM_CAT)
 ITEM_CAT = add_levenshtein_feature(ITEM_CAT, ['cat1', 'cat2'])
 MATRIX = MATRIX.merge(ITEM_CAT[['item_category_id', 'cat1_lab', 'cat2_lab', 'cat1_lev', 'cat2_lev']], how='left',
on='item_category_id')
<data_type_conversions>
train_features.cp_type.value_counts(normalize=True )
def pre_process_shop_data(df):
split = df.shop_name.str.split(n=1, expand=True)
 df['city'] = split[0].str.lower().str.strip()
df['city_id'] = df.city.factorize(sort=True)[0]
df['shopping_center'] = split[1].str.lower().str.contains('ТЦ'.lower())
<merge>
control_group = train_features.loc[train_features.cp_type == 'ctl_vehicle', 'sig_id']
train_targets_scored.loc[train_targets_scored['sig_id'].isin(control_group)].sum() [1:].sum()
pre_process_shop_data(SHOPS)
 MATRIX = MATRIX.merge(SHOPS[['shop_id', 'city_id', 'shopping_center']], how='left', on='shop_id' )<feature_engineering>
test_features.cp_type.value_counts(normalize=True )
def add_shop_revenue(df):
df['revenue'] = df.item_price * df.item_cnt_month
df['shop_revenue_month'] = df.groupby(['shop_id', 'date_block_num'])['revenue'].transform('sum')
 del df['revenue'] 
<feature_engineering>
train_features.cp_time.value_counts(normalize=True )
add_shop_revenue(MATRIX)
<merge>
train_features.cp_dose.value_counts(normalize=True )
def lag_feature(df, lags, col):

 tmp = df[['date_block_num', 'shop_id', 'item_lab', col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'shop_id', 'item_lab', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_lab'], how='left')
 df[col+'_lag_'+str(i)].fillna(0, inplace=True)
 return df<drop_column>
no_control_target = train_targets_scored.loc[~train_targets_scored['sig_id'].isin(control_group)]
MATRIX = lag_feature(MATRIX,(1, 2, 12), 'item_cnt_month')
 MATRIX = lag_feature(MATRIX,(1, 2, 12), 'shop_revenue_month')
 del MATRIX['shop_revenue_month'] 
<feature_engineering>
plt.plot(sorted(train_features.loc[train_features['sig_id'] == 'id_79fb45fe7', [col for col in train_features if 'g-' in col]].values.reshape(-1, 1)))
 plt.title('sorted g- value of id_79fb45fe7');
MATRIX['month'] = MATRIX.date_block_num.mod(12)
<feature_engineering>
s = pd.DataFrame({'sig_id': test_features['sig_id'].values} )
MATRIX['release'] = MATRIX.item_lab.map(
 MATRIX[MATRIX.item_cnt_month > 0].groupby(['item_lab'] ).agg(release=('date_block_num', 'min')).loc[:,'release'] 
)
 MATRIX['since_release'] = MATRIX.date_block_num - MATRIX.release
MATRIX['since_release'] = MATRIX.since_release.fillna(0)
 MATRIX = MATRIX[MATRIX.since_release >= 0]
del MATRIX['release'] 
<categorify>
s[train_targets_scored.columns[1:]] = 0.0001
MATRIX['mean_item_cnt_month'] = MATRIX.groupby('date_block_num')['item_cnt_month'].transform('mean')
 MATRIX[MATRIX.item_cnt_month == 34].mean_item_cnt_month = 0.28394 
<define_variables>
control_group = test_features.loc[test_features.cp_type == 'ctl_vehicle', 'sig_id']
IDX_TRAIN =(MATRIX.date_block_num > 11)&(MATRIX.date_block_num < 33)
 IDX_VAL = MATRIX.date_block_num == 33
IDX_TEST = MATRIX.date_block_num == 34<merge>
s.loc[s['sig_id'].isin(control_group), train_targets_scored.columns[1:]] = 0
def add_max_cnt(df):
max_cnt = df.loc[IDX_TRAIN, ['item_lab', 'shop_id', 'item_cnt_month']].groupby(['item_lab', 'shop_id'] ).agg(
 max_cnt=('item_cnt_month', 'max')).reset_index()
df = df.merge(max_cnt, how='left', on=['item_lab', 'shop_id'])
 return df<define_variables>
s.to_csv('basic_submission.csv', index=False )
MATRIX = add_max_cnt(MATRIX)
 fillna_means(MATRIX, 'max_cnt')
 IDX_TRAIN =(MATRIX.date_block_num > 11)&(MATRIX.date_block_num < 33)
 IDX_VAL = MATRIX.date_block_num == 33
IDX_TEST = MATRIX.date_block_num == 34<categorify>
train_features = pd.concat([train_features, pd.get_dummies(train_features['cp_time'], prefix='cp_time')], axis=1)
 train_features = pd.concat([train_features, pd.get_dummies(train_features['cp_dose'], prefix='cp_dose')], axis=1)
 train_features = pd.concat([train_features, pd.get_dummies(train_features['cp_type'], prefix='cp_type')], axis=1)
 train_features = train_features.drop(['cp_type', 'cp_time', 'cp_dose'], axis=1)
def get_mean_encodings_kfold(df, idx_train, idx_val, idx_test, feature_names, target_name, folds=5):
skf = StratifiedKFold(n_splits=folds, shuffle=True)
 global_mean = df.loc[idx_train, target_name].mean()
for feature_name in feature_names:
print(feature_name)
 df.loc[:, feature_name + '_mean'] = np.nan
data = df.loc[idx_train, [feature_name, feature_name + '_mean', target_name]]
for idx_1, idx_2 in skf.split(data[[feature_name]], data[feature_name]):
x_1 = data.iloc[idx_1].loc[:, [feature_name, feature_name + '_mean', target_name]]
x_2 = data.iloc[idx_2].loc[:, [feature_name, feature_name + '_mean', target_name]]
means = x_1.groupby(feature_name ).agg(mean_target=(target_name, 'mean')).loc[:, 'mean_target']
x_2[feature_name + '_mean'] = x_2[feature_name].map(means)
 data.update(x_2)
 data[feature_name + '_mean'].fillna(global_mean, inplace=True)
 df.update(data)
 for feature_name in feature_names:
x_1 = df.loc[idx_train, [feature_name, target_name]]
x_2 = df.loc[np.logical_or(idx_val, idx_test), [feature_name, target_name]]
means = x_1.groupby(feature_name ).agg(mean_target=(target_name, 'mean')).loc[:, 'mean_target']
x_2[feature_name + '_mean'] = x_2[feature_name].map(means)
 df.update(x_2)
 return df<categorify>
class MoADataset(Dataset):
def __init__(
 self,
data,
targets = None,
targets1 = None,
mode = 'train' 
):

 self.mode = mode
self.data = data
self.targets = targets
self.targets1 = targets1
def __getitem__(self, idx: int)-> Dict[str, np.array]:
data = self.data[idx]
if self.targets is not None:
target = self.targets[idx]
target1 = self.targets1[idx]
else:
target = np.zeros(( 206,))
target1 = np.zeros(( 402,))
sample = {'data': torch.tensor(data ).float() ,
'target': torch.tensor(target ).float() ,
'target1': torch.tensor(target1 ).float() }
return sample
def __len__(self)-> int:
return len(self.data )
MATRIX = get_mean_encodings_kfold(
 MATRIX, IDX_TRAIN, IDX_VAL, IDX_TEST,
['shop_id', 'city_id', 'item_lab', 'cat1_lab', 'cat2_lab', 'cat1_lev', 'cat2_lev', 'month'],
'item_cnt_month')
<prepare_x_and_y>
class MoADataModule(pl.LightningDataModule):
def __init__(self, hparams: Dict,
train_data, train_targets, train_targets1,
valid_data, valid_targets, valid_targets1):
super().__init__()
self.hparams = hparams
self.train_data = train_data
self.train_targets = train_targets
self.train_targets1 = train_targets1
self.valid_data = valid_data
self.valid_targets = valid_targets
self.valid_targets1 = valid_targets1
def prepare_data(self):
pass
def setup(self, stage=None):
self.train_dataset = MoADataset(data=self.train_data.iloc[:, 1:].values,
targets=self.train_targets.iloc[:, 1:].values,
targets1=self.train_targets1.iloc[:, 1:].values)
 self.valid_dataset = MoADataset(data=self.valid_data.iloc[:, 1:].values,
targets=self.valid_targets.iloc[:, 1:].values,
targets1=self.valid_targets1.iloc[:, 1:].values)
 def train_dataloader(self):
train_loader = torch.utils.data.DataLoader(
 self.train_dataset,
batch_size=1024,
num_workers=0,
shuffle=True, 
)
 return train_loader
def val_dataloader(self):
valid_loader = torch.utils.data.DataLoader(
 self.valid_dataset,
batch_size=1024,
num_workers=0,
shuffle=False, 
)
 return valid_loader
def test_dataloader(self):
return None
TRAIN_X = MATRIX[IDX_TRAIN].drop('item_cnt_month', axis=1)
 TRAIN_Y = MATRIX[IDX_TRAIN].item_cnt_month
VAL_X = MATRIX[IDX_VAL].drop('item_cnt_month', axis=1)
 VAL_Y = MATRIX[IDX_VAL].item_cnt_month
TEST_X = MATRIX[IDX_TEST].drop('item_cnt_month', axis=1 )<drop_column>
n_h_layers = 2048
learning_rate = 1e-3
criterion = nn.BCEWithLogitsLoss()
class Net(nn.Module):
def __init__(self, n_in, n_h, n_out, n_out1):
super(Net, self ).__init__()
self.fc1 = nn.Linear(n_in, n_h)
 self.fc2 = nn.Linear(n_h, math.ceil(n_h/4))
self.fc3 = nn.Linear(math.ceil(n_h/4), n_out)
 self.fc4 = nn.Linear(math.ceil(n_h/4), n_out1)
 self.bn = nn.BatchNorm1d(n_in)
 self.bn1 = nn.BatchNorm1d(n_h)
 self.bn2 = nn.BatchNorm1d(math.ceil(n_h/4))
self.drop = nn.Dropout(0.2)
 self.n_out = n_out
self.selu = nn.SELU()
self.sigm = nn.Sigmoid()
def forward(self, x, targets, targets1):
self.loss = criterion
x = self.fc1(self.bn(x))
x = self.selu(x)
 x = self.fc2(self.drop(self.bn1(x)))
 x = self.selu(x)
 x1 = self.fc3(self.bn2(x))
x2 = self.fc4(self.bn2(x))
loss =(self.loss(x1, targets)+ self.loss(x2, targets1)) / 2
real_loss = self.loss(x1, targets)
 out = self.sigm(x1)
 return out, loss, real_loss
del SALES, ITEM_CAT, ITEMS, SHOPS, TEST<set_options>
class LitMoA(pl.LightningModule):
def __init__(self, hparams, model):
super(LitMoA, self ).__init__()
self.hparams = hparams
self.model = model
def forward(self, x, targets, targets1, *args, **kwargs):
return self.model(x, targets, targets1)
 def configure_optimizers(self):
optimizer = torch.optim.AdamW(self.model.parameters() , lr=learning_rate, weight_decay=0.001)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=2)
 return(
 [optimizer],
[{'scheduler': scheduler, 'interval': 'epoch', 'monitor': 'valid_loss'}], 
)
 def training_step(
 self, batch: torch.Tensor, batch_idx: int 
)-> Union[int, Dict[str, Union[torch.Tensor, Dict[str, torch.Tensor]]]]:
data = batch['data']
target = batch['target']
target1 = batch['target1']
out, loss, real_loss = self(data, target, target1)
 logs = {'train_loss': loss, 'real_train_loss': real_loss}
return {
'loss': loss, 'real_train_loss': real_loss,
'log': logs,
'progress_bar': logs
}
def training_epoch_end(self, outputs):
avg_loss = torch.stack([x['loss'] for x in outputs] ).mean()
real_avg_loss = torch.stack([x['real_train_loss'] for x in outputs] ).mean()
logs = {'train_loss': avg_loss, 'real_train_loss': real_avg_loss}
return {'log': logs, 'progress_bar': logs}
def validation_step(
 self, batch: torch.Tensor, batch_idx: int 
)-> Union[int, Dict[str, Union[torch.Tensor, Dict[str, torch.Tensor]]]]:
data = batch['data']
target = batch['target']
target1 = batch['target1']
out, loss, real_loss = self(data, target, target1)
 logs = {'valid_loss': loss, 'real_valid_loss': real_loss}
return {
'loss': loss, 'real_valid_loss': real_loss,
'log': logs,
'progress_bar': logs,
}
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['loss'] for x in outputs] ).mean()
real_avg_loss = torch.stack([x['real_valid_loss'] for x in outputs] ).mean()
logs = {'valid_loss': avg_loss, 'real_valid_loss': real_avg_loss}
return {'valid_loss': avg_loss, 'log': logs, 'progress_bar': logs}
gc.collect()<train_model>
test_features = pd.concat([test_features, pd.get_dummies(test_features['cp_time'], prefix='cp_time')], axis=1)
 test_features = pd.concat([test_features, pd.get_dummies(test_features['cp_dose'], prefix='cp_dose')], axis=1)
 test_features = pd.concat([test_features, pd.get_dummies(test_features['cp_type'], prefix='cp_type')], axis=1)
 test_features = test_features.drop(['cp_type', 'cp_time', 'cp_dose'], axis=1)
 test_dataset = MoADataset(data=test_features.iloc[:, 1:].values)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=1024,
num_workers=0,
shuffle=False,
)
regressor = xgb.XGBRegressor(n_estimators = 100,
learning_rate = 0.08,
max_depth = 5,
subsample = 0.8,
colsample_bytree = 0.8,
n_jobs = 8 
)
 reg = regressor.fit(
 X=TRAIN_X,
y=TRAIN_Y,
eval_metric='rmse',
eval_set=[(TRAIN_X, TRAIN_Y),(VAL_X, VAL_Y)],
verbose=True,
early_stopping_rounds=8
)<load_pretrained>
class MetricsCallback(Callback):

 def __init__(self):
super().__init__()
self.metrics = []
def on_validation_end(self, trainer, pl_module):
self.metrics.append(trainer.callback_metrics )
pickle.dump(reg, open("xgb.pickle.dat", "wb"))<sort_values>
hparams = {}
importance.sort_values(ascending=False )<load_from_csv>
rkf = RepeatedKFold(n_splits=6, n_repeats=8, random_state=42)
 n = rkf.get_n_splits()
all_predictions = None
scores = []
for train_index, valid_index in rkf.split(train_features):
net = Net(n_in = 879, n_h = n_h_layers, n_out = 206, n_out1 = 402)
 train_data, valid_data = train_features.iloc[train_index, :], train_features.iloc[valid_index, :]
train_targets_scored_train, train_targets_scored_valid = train_targets_scored.iloc[train_index, :], train_targets_scored.iloc[valid_index, :]
train_targets_nonscored_train, train_targets_nonscored_valid = train_targets_nonscored.iloc[train_index, :], train_targets_nonscored.iloc[valid_index, :]
model = LitMoA(hparams=hparams, model=net)
 dm = MoADataModule(hparams=hparams,
train_data=train_data, train_targets=train_targets_scored_train, train_targets1=train_targets_nonscored_train,
valid_data=valid_data, valid_targets=train_targets_scored_valid, valid_targets1=train_targets_nonscored_valid)
 hparams = {}
metrics_callback = MetricsCallback()
trainer = pl.Trainer(
 early_stop_callback=EarlyStopping(monitor='valid_loss', patience=10, mode='min'),
checkpoint_callback=ModelCheckpoint(monitor='valid_loss', save_top_k=1, filepath='{epoch}_{valid_loss:.4f}', mode='min'),
gpus=1,
max_epochs=50,
log_save_interval=100,
num_sanity_val_steps=0,
gradient_clip_val=0.5,
weights_summary='full',
callbacks = [metrics_callback] 
)
 trainer.fit(model, dm)
 score = metrics_callback.metrics[-1]['real_valid_loss'].item()
scores.append(score)
 predictions = np.zeros(( test_features.shape[0], 206))
model_inference = model.model
model_inference.eval()
for ind, batch in enumerate(test_loader):
p = model_inference(batch['data'], batch['target'], batch['target1'])[0].detach().cpu().numpy()
predictions[ind * 1024:(ind + 1)* 1024] = p
if all_predictions is None:
all_predictions = predictions
else:
all_predictions += predictions
all_predictions = all_predictions / n
TEST = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 PRED = TEST_X[['item_lab', 'shop_id']]
PRED['item_cnt_month'] = reg.predict(TEST_X)
 PRED.drop_duplicates(inplace=True)
 print(TEST.shape, PRED.shape )<load_from_csv>
print(f'Mean score: {np.mean(scores):.4f}.Std score: {np.std(scores):.4f}' )
ITEMS = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 pre_process_item_data(ITEMS)
 TEST = TEST.merge(ITEMS[['item_id', 'item_lab']], how='left', on='item_id')
 print(TEST.shape )<merge>
s = pd.DataFrame({'sig_id': test_features['sig_id'].values} )
TEST = TEST.merge(PRED, how='left', on=['item_lab', 'shop_id'])
 TEST.shape<save_to_csv>
for col in train_targets_scored.columns[1:].tolist() :
s[col] = 0
TEST[['ID', 'item_cnt_month']].sort_values('ID' ).to_csv('submission.csv', index=False, header=True )<import_modules>
s.loc[:, train_targets_scored.columns[1:]] = all_predictions
<set_options>
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 s.loc[s['sig_id'].isin(test_features.loc[test_features['cp_type'] =='ctl_vehicle', 'sig_id']), train_targets_scored.columns[1:]] = 0
pd.set_option('display.float_format', lambda x: '%.5f' % x)
 %matplotlib inline 
<load_from_csv>
s.to_csv('submission.csv', index=False )