Kaggle competition: Mechanisms of Action (MoA) Prediction
df = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv' ).drop('id',axis=1)
 test = pd.read_csv(r'.. /input/cat-in-the-dat-ii/test.csv', index_col=0)
 sample_submission = pd.read_csv(r'.. /input/cat-in-the-dat-ii/sample_submission.csv', index_col=0 )<count_missing_values>
warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
( df.isna().sum(axis=1)==0 ).sum() /df.shape[0]<count_missing_values>
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42)
( test.isna().sum(axis=1)==0 ).sum() /test.shape[0]<count_missing_values>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_features.head()
df.isna().sum(axis=1 ).value_counts()<compute_train_metric>
train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )
def cochran_criterion(crosstab):
criterion = True
E = []
N = df.shape[0]
for O_i in crosstab.sum(axis=1):
for O_j in crosstab.sum() :
if O_i*O_j/N==0:
critetion = False
E.append(O_i*O_j/N > 4)
 criterion = criterion &(np.mean(E)>0.8)
 return criterion<compute_test_metric>
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape)
def chi2_test(X,Y):
crosstab = pd.crosstab(X, Y)
 criterion = cochran_criterion(crosstab)
 chi2, p = chi2_contingency(crosstab)[:2]
return [criterion, chi2, p]<compute_test_metric>
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x
start = datetime.now()
df_total = pd.concat([df.drop('target', axis=1), test])
 chi2_missingness = pd.DataFrame(columns=['Cochran_criterion', 'Chi2', 'p_value'])
 for col1 in df_total.columns:
for col2 in df_total.columns.drop(col1):
missingness_indicator = df_total[col1].isna()
other_variable = df_total[col2]
chi2_missingness.loc[col1 + '_' + col2] = chi2_test(missingness_indicator, other_variable)
 for col in df.columns.drop('target'):
missingness_indicator = df[col].isna()
chi2_missingness.loc[col + '_target'] = chi2_test(missingness_indicator, df.target)
 runtime = datetime.now() - start
print('Runtime ', runtime )<save_to_csv>
cat_features = ['cp_time', 'cp_dose']
num_features = [c for c in train.columns if train.dtypes[c] != 'object']
num_features = [c for c in num_features if c not in cat_features]
num_features = [c for c in num_features if c not in target_cols]
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test)
chi2_missingness.sort_values('p_value', inplace=True)
 reject, pvalue_corrected = fdrcorrection(pvals=chi2_missingness.p_value, alpha=0.05, method='indep', is_sorted=True)
 chi2_missingness['p_value_corrected'] = pvalue_corrected
chi2_missingness['BH_reject'] = chi2_missingness['Cochran_criterion'] & reject
chi2_missingness.to_csv('chi2_tests_missingness' )<filter>
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.5
lr=1e-3
weight_decay=1e-6
batch_size=32
epochs=21
num_features=num_features
cat_features=cat_features
target_cols=target_cols
significant_tests = chi2_missingness[chi2_missingness['BH_reject']==True]
print('Number of statistically significant tests: ', significant_tests.shape[0] )<split>
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x
def predict_missingness(col):
X = df_total.drop(col, axis=1)
 y = df_total[col].isna()
cols = X.columns.tolist()
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, stratify=y, random_state=0)
 X_train.reset_index(drop=True, inplace=True)
 y_train.reset_index(drop=True, inplace=True)
 X_train_te = X_train.copy()
for index_fit, index_transform in StratifiedKFold(n_splits=5, random_state=0, shuffle=True ).split(X_train, y_train):
target_enc = TargetEncoder(cols = cols, smoothing=0.5, min_samples_leaf=5)
 target_enc.fit(X_train.iloc[index_fit,:], y_train.iloc[index_fit])
 X_train_te.loc[index_transform,:] = target_enc.transform(X_train.iloc[index_transform, :])
 X_train_te = X_train_te.astype(float)
 target_enc = TargetEncoder(cols = cols, smoothing=0.5, min_samples_leaf=5)
 target_enc.fit(X_train, y_train)
 X_val_te = target_enc.transform(X_val)
 weight = sum(y_train==0)/sum(y_train==1)
 gbt = xgb.XGBClassifier(n_estimators=3000, random_state=0, objective='binary:logistic', scale_pos_weight=weight,
learning_rate=0.2, max_depth=4, subsample=0.8, colsample_bytree = 0.8, min_child_weight=1000)
 gbt.fit(X_train_te, y_train, eval_set= [(X_train_te, y_train),(X_val_te, y_val)], eval_metric=['auc'], early_stopping_rounds=50,
verbose=False)
 y_pred = gbt.predict_proba(X_train_te)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 y_pred = gbt.predict_proba(X_val_te)
 val_score = roc_auc_score(y_val, y_pred[:,1])
 return(train_score, val_score )<save_to_csv>
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
start = datetime.now()
pred_missingness = pd.DataFrame(columns=['Feature', 'train_score', 'val_score'])
 for col in df_total.columns:
train_score, val_score = predict_missingness(col)
 pred_missingness.loc[col] = [col, train_score, val_score]
pred_missingness.sort_values('val_score', ascending=False, inplace=True)
 pred_missingness.to_csv('pred_missingness.csv', index=False)
 runtime = datetime.now() - start
print('Runtime ', runtime )<categorify>
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr, weight_decay=cfg.weight_decay)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions
def predict_feature(col):
df_new = df_total[df_total[col].isna() ==False].astype(str)
 X = df_new.drop(col, axis=1)
 y = df_new[col]
le = LabelEncoder()
y = le.fit_transform(y)
 train_scores = []
val_scores = []
for index_train, index_val in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X, y):
X_train = X.iloc[index_train]
y_train = y[index_train]
X_valid = X.iloc[index_val]
y_valid = y[index_val]
oe = OrdinalEncoder()
X_train_oe = oe.fit_transform(X_train.astype(str))
X_val_oe = oe.transform(X_valid.astype(str))
X_train_oe += 2
X_val_oe += 2
nb = CategoricalNB()
nb.fit(X_train_oe, y_train)
 if np.max(y)==1:
y_pred = nb.predict_proba(X_train_oe)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 train_scores.append(train_score)
 y_pred = nb.predict_proba(X_val_oe)
 val_score = roc_auc_score(y_valid, y_pred[:,1])
 val_scores.append(val_score)
 else:
y_pred = nb.predict_proba(X_train_oe)
 train_score = roc_auc_score(y_train, y_pred, multi_class='ovr')
 train_scores.append(train_score)
 y_pred = nb.predict_proba(X_val_oe)
 val_score = roc_auc_score(y_valid, y_pred, multi_class='ovr')
 val_scores.append(val_score)
 return(train_scores, val_scores )<save_to_csv>
oof = np.zeros(( len(train), len(CFG.target_cols)))
 predictions = np.zeros(( len(test), len(CFG.target_cols)))
 SEED = [0, 1, 2]
for seed in SEED:
_oof, _predictions = run_kfold_nn(CFG,
train, test, folds,
num_features, cat_features, target,
device,
n_fold=5, seed=seed)
 oof += _oof / len(SEED)
 predictions += _predictions / len(SEED)
 score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"Seed Averaged CV score: {score}" )
start = datetime.now()
pred_features = pd.DataFrame(columns=['Feature', 'train_score', 'val_score', 'std_val_score'])
 high_cardinality_features = ['nom_' + str(i)for i in range(5,10)] + ['ord_5']
for col in df_total.columns.drop(high_cardinality_features):
train_score, val_score = predict_feature(col)
 pred_features.loc[col] = [col, np.mean(train_score), np.mean(val_score), np.std(val_score)]
pred_features.sort_values('val_score', ascending=False, inplace=True)
 pred_features.to_csv('pred_features.csv', index=False)
 runtime = datetime.now() - start
print('Runtime ', runtime )<count_values>
train[target_cols] = oof
train[['sig_id']+target_cols].to_csv('oof.csv', index=False)
 test[target_cols] = predictions
test[['sig_id']+target_cols].to_csv('pred.csv', index=False )
df.bin_4.fillna('NaN' ).value_counts()<count_values>
result = train_targets_scored.drop(columns=target_cols)\ 
.merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = result[target_cols].values
score = 0
for i in range(y_true.shape[1]):
_score = log_loss(y_true[:,i], y_pred[:,i])
 score += _score / y_true.shape[1]
logger.info(f"Final result: {score}" )
df.nom_5.fillna('NaN' ).value_counts()<prepare_x_and_y>
sub = submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()