Kaggle competition: Mechanisms of Action (MoA) Prediction
pre= pipe_2_gridsearch.predict(X_test )<save_to_csv>
warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": pre
})
 submission.to_csv('submission2_RF_pip_GS.csv', index=False )<define_search_space>
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
lr_pipe3 = Pipeline([ 
('sscaler2', StandardScaler()), 
('dt', DecisionTreeClassifier() 
)
 ])
 pipe_3_params = {'sscaler2__with_mean': [True, False],
'sscaler2__with_std': [True, False],
'dt__max_depth': [10],
'dt__random_state':[100],
'dt__max_features': [1, 3,5],
'dt__criterion':["gini"],
'dt__min_samples_leaf': [10],
'dt__min_samples_split': [2,3,4]}
pipe_3_gridsearch = GridSearchCV(lr_pipe3,
pipe_3_params,
cv=5,
verbose=1 , scoring="accuracy" , n_jobs=-1)
<train_model>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
pipe_3_gridsearch.fit(X_train, y_train);<find_best_score>
train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )
pipe_3_gridsearch.best_score_<predict_on_test>
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )
pre= pipe_3_gridsearch.predict(X_test )<save_to_csv>
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": pre
})
 submission.to_csv('submission2_DT_pip_GS.csv', index=False )<predict_on_test>
cat_features = ['cp_time', 'cp_dose']
num_features = [c for c in train.columns if train.dtypes[c] != 'object']
num_features = [c for c in num_features if c not in cat_features]
num_features = [c for c in num_features if c not in target_cols]
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )
tree = DecisionTreeClassifier(criterion='gini',max_depth=10,random_state=100,min_samples_leaf=10)
 tree.fit(X_train,y_train)
 y_predicted = tree.predict(Xs_test)
<compute_test_metric>
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.5
lr=1e-2
weight_decay=1e-6
batch_size=32
epochs=20
num_features=num_features
cat_features=cat_features
target_cols=target_cols
tree.score(Xs_train, y_train )<predict_on_test>
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x
pre= tree.predict(X_test )<save_to_csv>
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": pre
})
 submission.to_csv('submission2_DT.csv', index=False )<predict_on_test>
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr, weight_decay=cfg.weight_decay)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions
SVM = SVC()
SVM.fit(Xs_train, y_train)
 SVM_predictions = SVM.predict(Xs_test)
 SVM.score(Xs_train, y_train )<save_to_csv>
oof = np.zeros(( len(train), len(CFG.target_cols)))
 predictions = np.zeros(( len(test), len(CFG.target_cols)))
 SEED = [0, 1, 2]
for seed in SEED:
_oof, _predictions = run_kfold_nn(CFG,
train, test, folds,
num_features, cat_features, target,
device,
n_fold=5, seed=seed)
 oof += _oof / len(SEED)
 predictions += _predictions / len(SEED)
 score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"Seed Averaged CV score: {score}" )
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": SVM_predictions
})
 submission.to_csv('submission2_SVM.csv', index=False )<define_variables>
train[target_cols] = oof
train[['sig_id']+target_cols].to_csv('oof.csv', index=False)
 test[target_cols] = predictions
test[['sig_id']+target_cols].to_csv('pred.csv', index=False )
scores ={'LR_pip': 0.772,'LR': 0.770,'Knn_pip': 0.779, 'Knn': 0.669, 'RF_GS': 0.787,
'RF_pip_GS': 0.775, 'DT_pip_GS': 0.760, 'DT':0.779,
'SVM': 0.779}<set_options>
result = train_targets_scored.drop(columns=target_cols)\ 
.merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = result[target_cols].values
score = 0
for i in range(y_true.shape[1]):
_score = log_loss(y_true[:,i], y_pred[:,i])
 score += _score / y_true.shape[1]
logger.info(f"Final result: {score}" )
!jupyter nbextension enable --py widgetsnbextension<load_from_csv>
sub = submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()