Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<define_variables>
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
given_features = [
'prior_question_elapsed_time',
'prior_question_had_explanation',
'bundle_id',
]
deduced_features = [
'mean_user_accuracy',
'mean_accuracy_diff',
'mean_user_content_accuracy',
'answered_correctly_user',
'answered_user',
'mean_user_accuracy_r10',
'mean_user_accuracy_r25',
'mean_user_accuracy_r100',
'mean_content_accuracy',
'content_id_pos_diff',
'content_count',
'part',
'mean_user_part_accuracy',
'answered_part_user',
'tag_1',
'tag_2',
'hmean_user_content_accuracy',
'last_interaction_elapsed_time_l1',
'last_interaction_elapsed_time_l2',
'last_interaction_elapsed_time_l3',
'attempt',
'lectures_seen',
'last_question_mean_acc_diff',
'user_part_answered_consecutive',
'mean_user_mean_content_acc_diff',
'last_correct_time_elapsed',
'last_incorrect_time_elapsed',
]
features = given_features + deduced_features
features_df_cols = [
'user_id', 'content_id', 'part', 'tags',
'tags_label', 'answered_user',
'answered_correctly_user', 'mean_user_accuracy', 'mean_content_accuracy',
]
target = 'answered_correctly'
categorical_feature = ['part', 'retry', 'prior_question_had_explanation', 'bundle_id', 'tag_1', 'tag_2']
categorical_feature_idxs = []
for v in categorical_feature:
try:
categorical_feature_idxs.append(features.index(v))
except:
pass<data_type_conversions>
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
def get_train_val_idxs(TRAIN_SIZE, VAL_SIZE):
train_idxs = []
val_idxs = []
NEW_USER_FRAC = 1/4
np.random.seed(42)
 df = pd.DataFrame(index=get_index_np())
 for feature in ['user_id']:
df[feature] = load_feature(feature)
 df['index'] = df.index.values.astype(np.uint32)
 user_id_index = df.groupby('user_id')['index'].apply(np.array)
 for indices in user_id_index.sample(user_id_index.size, random_state=42):
if len(train_idxs)> TRAIN_SIZE:
break
if len(val_idxs)< VAL_SIZE:
if np.random.rand() < NEW_USER_FRAC:
val_idxs += list(indices)
 else:
offset = np.random.randint(0, indices.size)
 train_idxs += list(indices[:offset])
 val_idxs += list(indices[offset:])
 else:
train_idxs += list(indices)
 train_idxs = np.array(train_idxs, dtype=np.uint32)
 val_idxs = np.array(val_idxs, dtype=np.uint32)
 return train_idxs, val_idxs
train_idxs, val_idxs = get_train_val_idxs(int(96.5e6), 2.5e6)
 print(f'len train_idxs: {len(train_idxs)}, len validation_idxs: {len(val_idxs)}' )<prepare_x_and_y>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )
def make_x_y(train_idxs, val_idxs):
X_train = np.ndarray(shape=(len(train_idxs), len(features)) , dtype=np.float32)
 X_val = np.ndarray(shape=(len(val_idxs), len(features)) , dtype=np.float32)
 for idx, feature in enumerate(tqdm(features)) :
X_train[:,idx] = load_feature(feature, train_idxs, np.float32)
 X_val[:,idx] = load_feature(feature, val_idxs, np.float32)
 y_train = load_feature(target, train_idxs, np.int8)
 y_val = load_feature(target, val_idxs, np.int8)
 return X_train, y_train, X_val, y_val
X_train, y_train, X_val, y_val = make_x_y(train_idxs, val_idxs )<create_dataframe>
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )
display(pd.DataFrame(X_train[:25], columns=features))<create_dataframe>
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x
cat_features = ['cp_time', 'cp_dose']
num_features = ['g-0','g-2','g-3','g-5','g-6','g-7','g-8'
,'g-9','g-11','g-12','g-13','g-15','g-16','g-17','g-18','g-20','g-21'
,'g-22','g-24','g-25','g-26','g-27','g-28','g-29','g-30','g-31','g-32'
,'g-33','g-34','g-36','g-37','g-38','g-39','g-41','g-42','g-43','g-45'
,'g-47','g-48','g-49','g-50','g-51','g-52','g-53','g-54','g-55','g-56'
,'g-57','g-58','g-60','g-61','g-62','g-63','g-65','g-66','g-67','g-68'
,'g-69','g-70','g-71','g-72','g-73','g-75','g-76','g-77','g-78','g-79'
,'g-80','g-81','g-83','g-84','g-85','g-86','g-87','g-89','g-90','g-91'
,'g-92','g-93','g-94','g-96','g-97','g-98','g-100','g-101','g-102','g-103'
,'g-104','g-105','g-106','g-107','g-108','g-109','g-110','g-111','g-112'
,'g-113','g-114','g-115','g-116','g-117','g-118','g-119','g-120','g-121'
,'g-122','g-123','g-124','g-125','g-126','g-127','g-129','g-130','g-131'
,'g-132','g-133','g-134','g-135','g-136','g-137','g-138','g-139','g-140'
,'g-141','g-142','g-143','g-144','g-146','g-147','g-148','g-149','g-150'
,'g-151','g-152','g-154','g-156','g-157','g-158','g-160','g-161','g-162'
,'g-163','g-164','g-165','g-166','g-167','g-169','g-170','g-172','g-173'
,'g-174','g-175','g-177','g-178','g-179','g-180','g-181','g-183','g-184'
,'g-185','g-186','g-187','g-188','g-189','g-190','g-192','g-194','g-195'
,'g-196','g-199','g-200','g-202','g-203','g-205','g-206','g-207','g-208'
,'g-209','g-210','g-211','g-212','g-215','g-216','g-217','g-218','g-219'
,'g-221','g-222','g-224','g-225','g-226','g-227','g-228','g-229','g-230'
,'g-231','g-233','g-235','g-236','g-237','g-238','g-239','g-240','g-241'
,'g-242','g-243','g-245','g-246','g-247','g-248','g-250','g-251','g-252'
,'g-253','g-254','g-255','g-256','g-257','g-258','g-260','g-262','g-263'
,'g-265','g-267','g-268','g-269','g-270','g-272','g-273','g-274','g-276'
,'g-279','g-280','g-283','g-284','g-285','g-286','g-287','g-291','g-292'
,'g-293','g-294','g-296','g-297','g-298','g-299','g-300','g-301','g-302'
,'g-303','g-305','g-306','g-307','g-308','g-309','g-310','g-312','g-313'
,'g-314','g-317','g-318','g-319','g-321','g-322','g-323','g-324','g-325'
,'g-326','g-327','g-328','g-329','g-330','g-331','g-332','g-335','g-336'
,'g-337','g-338','g-340','g-341','g-342','g-343','g-344','g-346','g-347'
,'g-348','g-349','g-350','g-352','g-353','g-354','g-355','g-356','g-357'
,'g-358','g-359','g-360','g-361','g-362','g-363','g-365','g-366','g-367'
,'g-368','g-369','g-371','g-372','g-373','g-374','g-375','g-376','g-377'
,'g-379','g-380','g-381','g-382','g-383','g-384','g-385','g-386','g-387'
,'g-388','g-389','g-390','g-391','g-392','g-394','g-395','g-396','g-397'
,'g-398','g-400','g-402','g-403','g-404','g-405','g-407','g-408','g-409'
,'g-410','g-411','g-412','g-414','g-415','g-416','g-417','g-418','g-419'
,'g-420','g-421','g-422','g-423','g-424','g-425','g-426','g-427','g-428'
,'g-429','g-430','g-431','g-432','g-433','g-434','g-435','g-438','g-439'
,'g-440','g-441','g-442','g-443','g-444','g-445','g-446','g-447','g-449'
,'g-450','g-451','g-453','g-454','g-455','g-456','g-457','g-458','g-459'
,'g-460','g-461','g-462','g-463','g-465','g-466','g-468','g-469','g-470'
,'g-471','g-472','g-473','g-474','g-475','g-476','g-479','g-480','g-482'
,'g-483','g-484','g-485','g-486','g-488','g-489','g-491','g-492','g-493'
,'g-497','g-498','g-499','g-500','g-502','g-503','g-504','g-506','g-507'
,'g-508','g-509','g-510','g-511','g-513','g-514','g-515','g-516','g-518'
,'g-520','g-522','g-523','g-524','g-525','g-526','g-527','g-528','g-529'
,'g-530','g-531','g-533','g-534','g-535','g-536','g-537','g-538','g-539'
,'g-540','g-541','g-542','g-543','g-544','g-546','g-547','g-550','g-551'
,'g-552','g-553','g-554','g-555','g-556','g-557','g-558','g-559','g-560'
,'g-561','g-562','g-563','g-564','g-566','g-567','g-568','g-569','g-570'
,'g-571','g-572','g-574','g-577','g-578','g-579','g-580','g-583','g-584'
,'g-587','g-588','g-589','g-590','g-592','g-593','g-594','g-595','g-596'
,'g-597','g-598','g-599','g-600','g-602','g-604','g-605','g-606','g-608'
,'g-609','g-610','g-611','g-612','g-613','g-614','g-616','g-619','g-620'
,'g-622','g-624','g-627','g-628','g-629','g-630','g-631','g-632','g-634'
,'g-635','g-636','g-639','g-640','g-641','g-642','g-643','g-644','g-646'
,'g-647','g-648','g-649','g-651','g-652','g-655','g-656','g-657','g-658'
,'g-659','g-660','g-661','g-663','g-664','g-665','g-666','g-667','g-669'
,'g-671','g-672','g-673','g-674','g-675','g-677','g-678','g-679','g-681'
,'g-682','g-683','g-684','g-685','g-686','g-688','g-689','g-691','g-692'
,'g-693','g-694','g-696','g-697','g-698','g-699','g-700','g-701','g-702'
,'g-704','g-705','g-706','g-708','g-709','g-710','g-711','g-712','g-713'
,'g-714','g-720','g-722','g-724','g-725','g-726','g-727','g-728','g-729'
,'g-731','g-733','g-734','g-735','g-736','g-737','g-738','g-739','g-740'
,'g-741','g-742','g-743','g-744','g-745','g-746','g-747','g-748','g-749'
,'g-750','g-751','g-752','g-753','g-755','g-756','g-757','g-758','g-759'
,'g-760','g-761','g-762','g-763','g-764','g-766','g-767','g-768','g-769'
,'g-771','c-0','c-5','c-6','c-7','c-8','c-9','c-10','c-12','c-13','c-15'
,'c-18','c-20','c-22','c-24','c-25','c-26','c-30','c-33','c-34','c-36'
,'c-37','c-38','c-41','c-44','c-45','c-46','c-47','c-48','c-50','c-51'
,'c-52','c-54','c-56','c-57','c-58','c-59','c-60','c-62','c-63','c-64'
,'c-65','c-66','c-67','c-69','c-70','c-71','c-72','c-73','c-75','c-76'
,'c-77','c-79','c-80','c-81','c-83','c-85','c-86','c-87','c-89','c-92'
,'c-93','c-95','c-96','c-98','c-99']
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )
train_data = lgb.Dataset(
 data = X_train,
label = y_train,
categorical_feature = None, 
)
 val_data = lgb.Dataset(
 data = X_val,
label = y_val,
categorical_feature = None,
)<drop_column>
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.45
lr=1e-3
weight_decay=1e-5
batch_size=128
epochs=50
num_features=num_features
cat_features=cat_features
target_cols=target_cols
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions
def just_predict(cfg, test_loader, model_paths, device):
test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for path in model_paths:
model = TabularNN(cfg)
 model.load_state_dict(torch.load(path))
model.to(device)
 prediction = inference_fn(test_loader, model, device)
 predictions += prediction / len(model_paths)
 print('Done! ', path)
 return predictions
del X_train, y_train, X_val, y_val, train_idxs, val_idxs
gc.collect()<init_hyperparams>
model_paths = glob.glob('.. /input/moa-model-weights/pth_0908_dr45/*.pth')
 predictions = just_predict(CFG, test, model_paths, device)
 test[target_cols] = predictions
test
lgbm_params = {
'objective': 'binary',
'metric': METRICS,
'num_leaves': 200,
'learning_rate': 0.1,
}<train_model>
<save_to_csv><EOS> 
sub = submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub