Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<prepare_output>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
%%time
para_index = make_embed_index(PARAGRAM, word_index, vocab )<feature_engineering>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
def check_coverage(vocab, embeddings_index, word_index=None):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<compute_train_metric>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
print("Glove : ")
 oov_glove = check_coverage(vocab, glove_index, word_index)
 print("Paragram : ")
 oov_paragram = check_coverage(vocab, para_index, word_index)
 print("FastText : ")
 oov_fasttext = check_coverage(vocab, fasttext_index, word_index )<concatenate>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
embed_concat = np.concatenate(( make_embed_mat(fasttext_index), make_embed_mat(glove_index), make_embed_mat(para_index)) , axis=1 )<train_model>
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 train_drug = pd.read_csv(data_path + "train_drug.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 print(train.shape,test.shape)
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 print('Non-control:', train.shape,test.shape)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 print('Feature elimination by variance :', train.shape,test.shape)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
print("Rank Gauss:", train.shape,test.shape)
 if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
print("PCA:", train.shape,test.shape)
 if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
print("One Hot:", train.shape,test.shape)
 for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 print('Extra features:', train.shape,test.shape)
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
train = train.merge(targets, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 target_cols = [x for x in targets.columns if x != 'sig_id']
train_df = train
train_df.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 features_to_drop = ["sig_id", "cp_type"]
test.drop(features_to_drop, axis = 1, inplace = True)
 X_test = test.values
t_init = time.time()
print(f"Initialized in {(t_init - begin)// 60} minutes" )<categorify>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = [x for x in range(42,49)]
NFOLDS = 10
DRUG_THRESH = 18
train_df = make_cv_folds(train_df, SEEDS, NFOLDS, DRUG_THRESH)
 train_df.head()
class Noise(nn.Module):
def __init__(self, mean=0.0, stddev=0.1):
super(Noise, self ).__init__()
self.mean = mean
self.stddev = stddev
def forward(self, input):
noise = input.clone().normal_(self.mean, self.stddev)
 return input + noise<normalization>
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 10, min_lr = 1e-5, factor = 0.1),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(x.contiguous().view(-1, feature_dim), self.weight ).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )
class Model(nn.Module):
def __init__(self, embedding_matrix):
super(Model, self ).__init__()
h1 = 64
h2 = 32
hd = 32
self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.noise = Noise(stddev=0.05)
 self.lstm = nn.LSTM(embedding_matrix.shape[1], 2*h1, bidirectional=False, batch_first=True)
 self.gru = nn.GRU(2*h1, 2*h2, bidirectional=False, batch_first=True)
 self.lstm_att = Attention(2*h1, max_len)
 self.gru_att = Attention(2*h2, max_len)
 self.dropout = nn.Dropout(0.1)
 self.linear = nn.Linear(6*(h1+h2)+ nb_features, hd)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(hd, momentum=0.5)
 self.out = nn.Linear(hd, 1)
 def forward(self, x):
embed = self.embedding(x[0])
 if self.training:
embed = torch.squeeze(self.noise(torch.unsqueeze(embed, 0)))
 lstm, _ = self.lstm(embed)
 gru, _ = self.gru(lstm)
 att1 = self.lstm_att(lstm)
 att2 = self.gru_att(gru)
 avg_pool1 = torch.mean(lstm, 1)
 avg_pool2 = torch.mean(gru, 1)
 max_pool1, _ = torch.max(lstm, 1)
 max_pool2, _ = torch.max(gru, 1)
 conc = torch.cat(( att1, avg_pool1, max_pool1, att2, avg_pool2, max_pool2, x[1]), 1)
 conc = self.dropout(conc)
 conc = self.bn(self.relu(self.linear(conc)))
 conc = self.dropout(conc)
 out = self.out(conc)
 return out<compute_test_metric>
scores_auc_all = []
test_cv_preds = []
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
oof_all = np.zeros(( train_df.shape[0], targets[target_cols].shape[1]))
for s in SEEDS:
oof = np.zeros(( train_df.shape[0], targets[target_cols].shape[1]))
tabnet_params['seed'] = s
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 kfold_col = f'kfold_{s}'
trn_idx = train_df[train_df[kfold_col] != fold_id].index
val_idx = train_df[train_df[kfold_col] == fold_id].index
X_train, y_train = train_df[feature_cols].values[trn_idx, :], targets[target_cols].values[trn_idx, :]
X_val, y_val = train_df[feature_cols].values[val_idx, :], targets[target_cols].values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 30,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 saving_path_name = 'TabNet_seed_'+str(tabnet_params['seed'])+'_fold_'+str(fold_id+1)
 saved_filepath = model.save_model(saving_path_name)
 print(saved_filepath)
 loaded_model = TabNetRegressor()
loaded_model.load_model(saving_path_name+'.zip')
 preds_val = loaded_model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof[val_idx] = preds
oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_all += oof
oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds)
 oof_all = oof_all/len(SEEDS )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<find_best_params>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
def get_lr(epoch):
if epoch <= 3:
return 0.001
else:
return 0.0005<categorify>
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()
def tweak_threshold(train_preds, y_train):
tmp = [0,0,0]
delta = 0
for tmp[0] in np.arange(0.1, 0.501, 0.01):
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
return tmp[2], delta<predict_on_test>
np.save('oof_TabNet_no_ctrl.npy', oof_all )
def predict(X, f,model, batch_size=1024):
y = np.array([])
 X = torch.tensor(X, dtype=torch.long ).cuda()
f = torch.tensor(f, dtype=torch.float32 ).cuda()
Xf = torch.utils.data.TensorDataset(X, f)
 loader = torch.utils.data.DataLoader(Xf, batch_size=batch_size, shuffle=False)
 for i,(x, f)in enumerate(loader):
y = np.concatenate(( y, sigmoid(model([x, f] ).detach().cpu().numpy())[:, 0]))
return y<train_model>
train_targets_scored = pd.read_csv(data_path + "train_targets_scored.csv")
 target_cols = [col for col in train_targets_scored.columns if col != 'sig_id']
def fit(model, X_train, f_train, y_train, X_val=None, f_val=None, y_val=None, epochs=5, batch_size=512):
history = {"loss":[], "val_loss": []}
best_loss = 10
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001)
 X_train = torch.tensor(X_train, dtype=torch.long ).cuda()
f_train = torch.tensor(f_train, dtype=torch.float32 ).cuda()
y_train = torch.tensor(y_train[:, np.newaxis], dtype=torch.float32 ).cuda()
train = torch.utils.data.TensorDataset(X_train, f_train, y_train)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 X_val = torch.tensor(X_val, dtype=torch.long ).cuda()
f_val = torch.tensor(f_val, dtype=torch.float32 ).cuda()
y_val = torch.tensor(y_val[:, np.newaxis], dtype=torch.float32 ).cuda()
val = torch.utils.data.TensorDataset(X_val, f_val, y_val)
 val_loader = torch.utils.data.DataLoader(val, batch_size=batch_size, shuffle=False)
 for epoch in range(epochs):
model.train()
avg_loss = 0
start_time = time.time()
lr = get_lr(epoch)
 for param_group in optimizer.param_groups:
param_group['lr'] = lr
for x_batch, f_batch, y_batch in train_loader:
optimizer.zero_grad()
y_pred = model([x_batch, f_batch])
 loss = loss_fn(y_pred, y_batch)
 loss.backward()
optimizer.step()
avg_loss += loss.item() / X_train.shape[0]
model.eval()
avg_val_loss = 0.
 for i,(x_batch, f_batch, y_batch)in enumerate(val_loader):
y_pred = model([x_batch, f_batch] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / X_val.shape[0]
history['loss'].append(avg_loss)
 history['val_loss'].append(avg_val_loss)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t lr={} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, epochs, lr, avg_loss, avg_val_loss, elapsed_time))
return history<split>
train = pd.read_csv(data_path + 'train_features.csv')
 print(train.shape)
 train_no_ctrl = train[train["cp_type"] != "ctl_vehicle"].reset_index(drop=True)
 print(train_no_ctrl.shape)
 train_pred = pd.DataFrame(oof_all, columns = target_cols)
 print(train_pred.shape)
 train_pred = pd.concat([train_no_ctrl[['sig_id']], train_pred], axis=1 ).reset_index(drop=True)
 print(train_pred.shape)
 display(train_pred.head() )
def k_fold(model_class, embedding_matrix, X, f, y, X_test, f_test, k=5, batch_size=512, epochs=5, seed=seed):
splits = list(StratifiedKFold(n_splits=k, shuffle=True, random_state=420 ).split(X, y))
pred_test = np.zeros(X_test.shape[0])
 pred_oof = np.zeros(y.shape[0])
 histories = []
for i,(train_idx, val_idx)in enumerate(splits):
print(f"------------- Fold {i+1} -------------
")
 seed_everything(seed + i)
 cp_path = f"{i}_weights.pth.tar"
start_time = time.time()
model = model_class(embedding_matrix)
 history = fit(model, X[train_idx], f[train_idx], y[train_idx], X[val_idx], f[val_idx], y[val_idx], epochs=epochs, batch_size=batch_size)
 histories.append(history)
 pred_oof[val_idx] = predict(X[val_idx], f[val_idx], model)
 pred_test += predict(X_test, f_test, model)/ k
score, threshold = tweak_threshold(pred_oof[val_idx], y[val_idx])
 print(f"
Scored {score :.4f} for threshold {threshold :.3f} on validation data")
 print(f"
Done in {(time.time() - start_time)/ 60 :.1f} minutes
")
 return pred_test, pred_oof, histories<split>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train_pred, on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
np.save('oof_TabNet_all.npy', y_pred )
pred_test, pred_oof, histories = k_fold(Model, embed_concat,
X_train, features_train, y_train, X_test, features_test,
k=5, batch_size=512, epochs=4 )<compute_test_metric>
def log_loss_numpy(y_pred, y_true):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()
<data_type_conversions><EOS> 
y_true = train_targets_scored[target_cols].values
log_loss_numpy(y_pred, y_true )