Kaggle competition: Mechanisms of Action (MoA) Prediction
print(f"OOF AUC : {round(roc_auc_score(train.target.values, oof_pred_deepfm), 5)}" )<save_to_csv>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
test_idx = test.id.values
submission = pd.DataFrame.from_dict({
'id': test_idx,
'target': y_pred_deepfm
})
 submission.to_csv("submission.csv", index=False)
 print("Submission file saved!" )<save_model>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
np.save('oof_pred_deepfm.npy',oof_pred_deepfm)
 np.save('y_pred_deepfm.npy', y_pred_deepfm )<load_from_csv>
def preprocessor_TabNet() :
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 train_drug = pd.read_csv(data_path + "train_drug.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
if decompo == "PCA":
print(b_, "PCA")
 pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
train = train.merge(targets, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 target_cols = [x for x in targets.columns if x != 'sig_id']
train_df = train
train_df.reset_index(drop=True, inplace=True)
 test.reset_index(drop = True, inplace = True)
 features_to_drop = ["sig_id", "cp_type"]
test.drop(features_to_drop, axis = 1, inplace = True)
 feature_cols = [c for c in train_df.columns if c not in targets.columns]
feature_cols = [c for c in feature_cols if c not in [ 'sig_id', 'drug_id', 'cp_type']]
return train_df, targets, test, submission, target_cols, feature_cols
%%time
train = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/test.csv" )<concatenate>
SEEDS = [x for x in range(42,49)]
NFOLDS = 10
X_test = test_TabNet.values
test["target"] = -1
data = pd.concat([train, test] ).reset_index(drop=True )<categorify>
MAX_EPOCH = 200
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 p_min = 0.0005
p_max = 0.9995
def InferenceTabNet(X_test, SEEDS, NFOLDS,DEVICE):
test_cv_preds = []
for s in SEEDS:
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', s)
 print(g_, '*' * 60, c_)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/TabNet_seed_'\
+str(s)+'_fold_'+str(fold_id+1)
 saved_file = '/kaggle/working/model'
shutil.make_archive(saved_file,'zip',saved_path_name)
 print(f'Successfully saved model at {saved_file}.zip')
 with zipfile.ZipFile('model.zip')as z:
with z.open('model_params.json')as f:
loaded_params = json.load(f)
 with z.open('network.pt')as f:
try:
saved_state_dict = torch.load(f, map_location=DEVICE)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.bytesIO(f.read()),map_location = DEVICE)
 loaded_model = TabNetRegressor(**loaded_params)
 loaded_model._set_network()
loaded_model.network.load_state_dict(saved_state_dict)
 loaded_model.network.eval()
preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/'
oof_TabNet_all = np.load(saved_path_name + 'oof_TabNet_all.npy')
 return oof_TabNet_all,test_preds_all
oof_TabNet_all,test_preds_all = InferenceTabNet(X_test, SEEDS, NFOLDS,DEVICE )
for feature in features:
train[feature] = train[feature].map(feature_map[feature])
 test[feature] = test[feature].map(feature_map[feature] )<set_options>
def to_submission_TabNet(test_preds_all,submission_TabNet):
data_path = ".. /input/lish-moa/"
all_feat = [col for col in submission_TabNet.columns if col not in ["sig_id"]]
test_TabNet = pd.read_csv(data_path + "test_features.csv")
 sig_id = test_TabNet[test_TabNet["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission_TabNet = pd.merge(test_TabNet[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission_TabNet.fillna(0, inplace = True)
 submission_TabNet.to_csv("submission_TabNet.csv", index = None)
 return submission_TabNet
submission_TabNet = to_submission_TabNet(test_preds_all,submission_TabNet)
 display(submission_TabNet.head())
 print(f"{b_}submission_TabNet.shape: {r_}{submission_TabNet.shape}" )
warnings.filterwarnings("ignore" )<compute_test_metric>
%who DataFrame
del targets_TabNet, test_TabNet, train_df,target_cols, feature_cols
gc.collect()
%who DataFrame
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<choose_model_class>
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed)
 def preprocessor_nn_transfer() :
variance_threshould = 0.8
ncompo_genes = 600
ncompo_cells = 50
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 train_features = train_features[train_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_features = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_features = train_features.drop('cp_type', axis=1)
 test_features = test_features.drop('cp_type', axis=1)
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train_features[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train_features[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train_features = pd.concat([train_features, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test_features[GENES])
 pca_cells_test = pca_cells.transform(test_features[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test_features = pd.concat([test_features, pca_genes_test, pca_cells_test], axis = 1)
 PCA')
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
mask =(train_features[cols_numeric].var() >= variance_threshould ).values
tmp = train_features[cols_numeric].loc[:, mask]
train_features = pd.concat([train_features[["sig_id", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
test_features = pd.concat([test_features[["sig_id", "cp_time", "cp_dose"]], test_features.loc[:,cols_numeric]], axis = 1)
 train_features = pd.get_dummies(train_features, columns = ['cp_time', 'cp_dose'])
 test_features = pd.get_dummies(test_features, columns = ['cp_time', 'cp_dose'])
 Feature selection')
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 test = test_features
Join')
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 num_targets: {}'.format(num_targets))
')
 return train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols
def create_model() :
inp = layers.Input(shape=(23,))
x = layers.Embedding(5725, 10 )(inp)
 x = layers.Flatten()(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(500, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 y = layers.Dense(2, activation="softmax" )(x)
 model = Model(inputs=inp, outputs=y)
 return model<choose_model_class>
train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols = preprocessor_nn_transfer()
model = create_model()<define_variables>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
oof_preds = np.zeros(( len(train_data)))
 test_preds = np.zeros(( len(test_data)) )<split>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
SEEDS = list(range(42,49))
NFOLDS = 10
DRUG_THRESH = 18
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
%%time
skf = StratifiedKFold(n_splits=10)
 k = 0
for train_index, test_index in skf.split(train_data, train_label):
x_train, x_test = train_data.iloc[train_index, :], train_data.iloc[test_index, :]
y_train, y_test = train_label.iloc[train_index, :], train_label.iloc[test_index, :]
model = create_model()
adam = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.999, amsgrad=False)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=[auc])
 lr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.1,
patience=3, min_lr=1e-6, mode='max', verbose=1)
 es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,
verbose=1, mode='max', baseline=None, restore_best_weights=True)
 model.fit(x_train,
utils.to_categorical(y_train),
validation_data=(x_test, utils.to_categorical(y_test)) ,
verbose=1,
batch_size=1024,
callbacks=[lr, es],
epochs=20 
)
 valid_fold_preds = model.predict(x_test)[:, 1]
test_fold_preds = model.predict(test_data)[:, 1]
oof_preds[test_index] = valid_fold_preds.ravel()
test_preds += test_fold_preds.ravel()
print("KFold %d, the best auc is %.4f" %(k, metrics.roc_auc_score(y_test, valid_fold_preds)))
 k += 1
K.clear_session()<save_to_csv>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 test_ = test
model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transferlearning-7seeds4248-10folds/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return predictions
def run_k_fold(NFOLDS, seed_id):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
return predictions
test_preds /= 10
test_ids = test.id.values
print("Saving submission file")
 submission = pd.DataFrame.from_dict({
'id': test_ids,
'target': test_preds
})
 submission.to_csv("submission.csv", index=False )<set_options>
predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
def Inference_Transfer_learning(predictions,SEEDS,NFOLDS):
for seed_id in SEEDS:
predictions_ = run_k_fold(NFOLDS, seed_id)
 predictions += predictions_ / len(SEEDS)
 return predictions
predictions = Inference_Transfer_learning(predictions,SEEDS,NFOLDS)
 time_diff = time() - time_begin
test[target_cols] = predictions
str(timedelta(seconds=time_diff))
!pip install --upgrade scikit-learn
py.init_notebook_mode(connected=True)
<load_from_csv>
def to_submission_Transfer_learning(target_cols,test):
data_dir = '.. /input/lish-moa/'
sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 sub_nn_transfer = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub_nn_transfer.to_csv('submission_nn_transfer.csv', index=False)
 return sub_nn_transfer
sub_nn_transfer = to_submission_Transfer_learning(target_cols,test)
 print(sub_nn_transfer.shape)
 display(sub_nn_transfer.head() )
path='.. /input/cat-in-the-dat-ii/'
train=pd.read_csv(path+'train.csv')
 test=pd.read_csv(path+'test.csv')
 submission=pd.read_csv(path+'sample_submission.csv')
<prepare_output>
%who DataFrame
del test, train
gc.collect()
%who DataFrame
def resumetable(df):
print(f"Dataset Shape: {df.shape}")
 summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Total'] = df.count().values
summary['Missing Percentage']=(summary['Missing']/summary['Total'])*100
summary['Uniques'] = df.nunique().values
return summary<define_variables>
def readin_data_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets_drug = train_targets_scored.merge(train_drug, on='sig_id')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
targets_drug = targets_drug.iloc[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/ttestpcarfelogisticregression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,labels_train,test_features,predictors,cs,gs
bin_features=[i for i in train.columns if i.split('_')[0]=='bin']
ord_features=[i for i in train.columns if i.split('_')[0]=='ord']
nom_features=[i for i in train.columns if i.split('_')[0]=='nom']
cyc_features=[i for i in train.columns if i in ['day','month']]<correct_missing_values>
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def resnet_preprocessor(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
train_copy=train.dropna()<compute_test_metric>
def resnet_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.25),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model
def cramers_v(x, y):
confusion_matrix = pd.crosstab(x,y)
 chi2 = ss.chi2_contingency(confusion_matrix)[0]
n = confusion_matrix.sum().sum()
phi2 = chi2/n
r,k = confusion_matrix.shape
phi2corr = max(0, phi2-(( k-1)*(r-1)) /(n-1))
rcorr = r-(( r-1)**2)/(n-1)
 kcorr = k-(( k-1)**2)/(n-1)
 return np.sqrt(phi2corr/min(( kcorr-1),(rcorr-1)))
 all_feat=bin_features+nom_features+ord_features+cyc_features<categorify>
def predict_resnet(train_features,test_features,labels_train, SEEDS, NFOLDS,cs,gs):
n_labels = labels_train.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
n_seeds = len(SEEDS)
 n_folds = NFOLDS
y_pred = np.zeros(( n_test,n_labels))
_,data_test = resnet_preprocessor(train_features.values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 data_test_2 = test_features[predictors].values
for seed in SEEDS:
for fold in range(NFOLDS):
K.clear_session()
model = keras.models.load_model('/kaggle/input/resent-1841/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold), custom_objects={'logloss':logloss})
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 print(f"Pretrained_Resnet_Model: {seed}, FOLD: {fold} ")
 return y_pred
def encoding(train, test, smooth):
print('Target encoding...')
 train.sort_index(inplace=True)
 target = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 cat_feat_to_encode = bin_features+nom_features+ord_features+cyc_features
smoothing=smooth
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(train, target):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], target.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, target)
 train = oof.sort_index()
test = ce_target_encoder.transform(test)
 features = list(train)
 print('Target encoding done!')
 return train, test, test_id, features, target 
<categorify>
train_features,labels_train,test_features,predictors,cs,gs = readin_data_resnet()
p_min = 0.0005
p_max = 0.9995
DRUG_THRESH = 18
n_seeds = 7
np.random.seed(42)
 SEEDS = np.random.randint(0,100,size=n_seeds)
 NFOLDS = 10
test_pred_rs = predict_resnet(train_features,test_features,labels_train,SEEDS, NFOLDS,cs,gs )
train_encode, test_encode, test_id, features, target = encoding(train, test, 0.3 )<concatenate>
sub_rs = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_rs.iloc[:,1:] = np.zeros([test_pred_rs.shape[0],test_pred_rs.shape[1]])
 sub_rs.iloc[:,1:] = test_pred_rs
sub_rs.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_rs.to_csv('submission_rs.csv', index=False)
train_encode=pd.concat([train_encode,target],axis=1,ignore_index=True)
 train_encode.columns=list(train.columns)+['target']<prepare_x_and_y>
%who DataFrame
del train_features,labels_train,test_features,predictors,cs,gs
gc.collect()
%who DataFrame
X, y = train_encode[all_feat], train_encode['target']<choose_model_class>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def remove_ctl(no_ctl,train,test,targets):
if no_ctl:
print('not_ctl')
 train = train[train['cp_type']!='ctl_vehicle']
test = test[test['cp_type']!='ctl_vehicle']
targets = targets.iloc[train.index]
train.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 targets.reset_index(drop=True, inplace=True)
 return train,test,targets
def remove_small_variance(variance_threshould,train,test):
cols_numeric = [feat for feat in list(train.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp_train = train[cols_numeric].loc[:, mask]
tmp_test = test[cols_numeric].loc[:, mask]
train = pd.concat([train[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_train], axis=1)
 test = pd.concat([test[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_test], axis=1)
 return train,test
def pca_process(ncompo_genes,ncompo_cells,train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
pca_genes = PCA(n_components=ncompo_genes, random_state=base_seed ).fit(train[GENES])
 pca_cells = PCA(n_components=ncompo_cells, random_state=base_seed ).fit(train[CELLS])
 train_pca_gene = pca_genes.transform(train[GENES])
 test_pca_gene = pca_genes.transform(test[GENES])
 train_pca_cell = pca_cells.transform(train[CELLS])
 test_pca_cell = pca_cells.transform(test[CELLS])
 train_pca_gene = pd.DataFrame(train_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 test_pca_gene = pd.DataFrame(test_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 train_pca_cell = pd.DataFrame(train_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 test_pca_cell = pd.DataFrame(test_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 train_df = pd.concat([train, train_pca_gene, train_pca_cell], axis=1)
 test_df = pd.concat([test, test_pca_gene, test_pca_cell], axis=1)
 return train_df,test_df
def scale_data(scale, train_df,test_df):
if scale == 'minmax':
print('minmax')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 elif scale == 'rankgauss':
print('rankgauss')
 cols_numeric = [feat for feat in list(train_df.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose'] and "pca_" not in feat]
scaler = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train_df[cols_numeric] = scaler.fit_transform(train_df[cols_numeric])
 test_df[cols_numeric] = scaler.transform(test_df[cols_numeric])
 else:
pass
return train_df,test_df
def add_features(train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
for stats in tqdm.tqdm(['sum', 'mean', 'std', 'kurt', 'skew']):
train['g_'+stats] = getattr(train[GENES], stats )(axis=1)
 train['c_'+stats] = getattr(train[CELLS], stats )(axis=1)
 train['gc_'+stats] = getattr(train[GENES+CELLS], stats )(axis=1)
 test['g_'+stats] = getattr(test[GENES], stats )(axis=1)
 test['c_'+stats] = getattr(test[CELLS], stats )(axis=1)
 test['gc_'+stats] = getattr(test[GENES+CELLS], stats )(axis=1)
 return train,test
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def data_processing(train,test, targets):
train,test,targets = remove_ctl(no_ctl,train,test,targets)
 print("remove control: ", train.shape,test.shape)
 train,test = remove_small_variance(variance_threshould,train,test)
 print("remove small_var: ", train.shape,test.shape)
 train_df,test_df = pca_process(ncompo_genes,ncompo_cells,train,test)
 print("do PCA: ", train_df.shape,test_df.shape)
 train_df,test_df = scale_data(scale, train_df,test_df)
 print("rankGauss: ", train_df.shape,test_df.shape)
 train_df,test_df = add_features(train_df,test_df)
 print("add stats feature:", train_df.shape,test_df.shape)
 return train_df,test_df,targets
def make_classifier() :
clf = CatBoostClassifier(
 loss_function='CrossEntropy',
eval_metric="AUC",
task_type="CPU",
learning_rate=0.05,
n_estimators =100,
early_stopping_rounds=10,
random_seed=2019,
silent=True 
)
 return clf 
<init_hyperparams>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
scoring = "roc_auc"
HistGBM_param = {
'l2_regularization': 0.0,
'loss': 'auto',
'max_bins': 255,
'max_depth': 15,
'max_leaf_nodes': 31,
'min_samples_leaf': 20,
'n_iter_no_change': 50,
'scoring': scoring,
'tol': 1e-07,
'validation_fraction': 0.15,
'verbose': 0,
'warm_start': False
}
folds = StratifiedKFold(n_splits=3, shuffle=True, random_state=1)
 fold_preds = np.zeros([test_encode.shape[0],3])
 oof_preds = np.zeros([X.shape[0],3])
 results = {}
estimators = [ 
('histgbm', HistGradientBoostingClassifier(**HistGBM_param)) , 
('catboost', make_classifier())
 ]
f, ax = plt.subplots(1,3,figsize = [14,5])
 for i,(trn_idx, val_idx)in enumerate(folds.split(X,y)) :
print(f"Fold {i} stacking.... ")
 clf = StackingClassifier(
 estimators=estimators,
final_estimator=LogisticRegression() , 
)
 clf.fit(X.loc[trn_idx,:], y.loc[trn_idx])
 tmp_pred = clf.predict_proba(X.loc[val_idx,:])[:,1]
oof_preds[val_idx,0] = tmp_pred
fold_preds[:,0] += clf.predict_proba(test_encode)[:,1] / folds.n_splits
estimator_performance = {}
estimator_performance['stack_score'] = metrics.roc_auc_score(y.loc[val_idx], tmp_pred)
 for ii, est in enumerate(estimators):
model = clf.named_estimators_[est[0]]
pred = model.predict_proba(X.loc[val_idx,:])[:,1]
oof_preds[val_idx, ii+1] = pred
fold_preds[:,ii+1] += model.predict_proba(test_encode)[:,1] / folds.n_splits
estimator_performance[est[0]+"_score"] = metrics.roc_auc_score(y.loc[val_idx], pred)
 stack_coefficients = {x+"_coefficient":y for(x,y)in zip([x[0] for x in estimators], clf.final_estimator_.coef_[0])}
stack_coefficients['intercept'] = clf.final_estimator_.intercept_[0]
results["Fold {}".format(str(i+1)) ] = [
estimator_performance,
stack_coefficients
]
plot_roc_curve(clf, X.loc[val_idx,:], y.loc[val_idx], ax=ax[i])
 ax[i].plot([0.0, 1.0])
 ax[i].set_title("Fold {} - ROC AUC".format(str(i)))
 plt.tight_layout(pad=2)
 plt.show()
f, ax = plt.subplots(1,2,figsize = [11,5])
 sns.heatmap(pd.DataFrame(oof_preds, columns = ['stack','histgbm','catboost'] ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="magma",ax=ax[0])
 ax[0].set_title("OOF PRED - Correlation Plot")
 sns.heatmap(pd.DataFrame(fold_preds, columns = ['stack','histgbm','catboost'] ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="inferno",ax=ax[1])
 ax[1].set_title("TEST PRED - Correlation Plot")
 plt.tight_layout(pad=3)
 plt.show()<save_to_csv>
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.8
decompo = 'PCA'
ncompo_genes = 50
ncompo_cells = 8
encoding = 'dummy'
base_seed = 42
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 20
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=1024
SEEDS = 7
SEED = [0, 1, 2, 3, 4, 5, 6]
DRUG_THRESH = 18
submission['target'] =fold_preds[:,0]
submission.to_csv('submission.csv', index=None)
 submission.head()<install_modules>
seed_everything(base_seed)
 def preprocessor_Multi_heads() :
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, test, targets = data_processing(train_features,test_features, train_targets_scored)
 print('length of target_cols:',len(target_cols))
train_comb = train.merge(targets, on='sig_id')
 train_comb.drop('cp_type', axis=1, inplace=True)
 print("combine train and targets to prepare for kfolds ", train_comb.shape)
 feature_cols = [c for c in process_data(train_comb ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
print('length of feature_cols:',len(feature_cols))
gen_cols = [c for c in feature_cols if 'g-' == c[:2]]
cell_cols = [c for c in feature_cols if 'c-' == c[:2]]
pca_gen_cols = [c for c in feature_cols if 'pca_g' == c[:5]]
pca_cell_cols = [c for c in feature_cols if 'pca_c' == c[:5]]
ohe_cols = [c for c in feature_cols if c not in gen_cols+cell_cols+pca_gen_cols+pca_cell_cols]
print(len(gen_cols), len(cell_cols), len(pca_gen_cols), len(pca_cell_cols),len(ohe_cols))
num_features=len(feature_cols)
 num_targets=len(target_cols)
 folds = train_comb.copy()
folds = folds.merge(train_drug, on='sig_id')
 folds = make_cv_folds(folds, SEEDS, NFOLDS, DRUG_THRESH)
 return folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols,test_features
folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols, test_features = preprocessor_Multi_heads()
pip install --upgrade scikit-learn<import_modules>
class Model(nn.Module):
def __init__(self, num_gen_features, num_cell_features,num_targets, hidden_size):
super(Model, self ).__init__()
self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(0.2)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(0.3)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(0.2)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, int(hidden_size/2)))
 self.cell_batch_norm2 = nn.BatchNorm1d(int(hidden_size/2))
self.cell_dropout2 = nn.Dropout(0.3)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(int(hidden_size/2), int(hidden_size/2)))
 self.batch_norm3 = nn.BatchNorm1d(hidden_size + int(hidden_size/2))
self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size + int(hidden_size/2), num_targets))
def forward(self, gen_x, cell_x):
gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.cell_dense2(cell_x))
x = torch.cat(( gen_x,cell_x),dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
print(sklearn.__version__ )<import_modules>
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float)
 }
return dct
notebookstart = time.time()
@contextmanager
def timer(name):

 t0 = time.time()
yield
print('
[{}] done in {} Minutes'.format(name, round(( time.time() - t0)/60,2)) )<define_variables>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
gen_x, cell_x = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(gen_x, cell_x)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
seed = 50
debug = False
if debug:
nrow = 5000
else:
nrow = None<load_from_csv>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
with timer("Load"):
PATH = "/kaggle/input/cat-in-the-dat-ii/"
train = pd.read_csv(PATH + "train.csv", index_col = 'id', nrows = nrow)
 test = pd.read_csv(PATH + "test.csv", index_col = 'id')
 submission_df = pd.read_csv(PATH + "sample_submission.csv")
 [print(x.shape)for x in [train, test, submission_df]]
traindex = train.index
testdex = test.index
y = train.target.copy()
print("Target Distribution:
",y.value_counts(normalize = True ).to_dict())
 df = pd.concat([train.drop('target',axis = 1), test], axis = 0)
 del train, test, submission_df<feature_engineering>
def get_oof_predictions(fold,seed):
seed_everything(seed)
 test_ = process_data(test)
 model = Model(
 num_gen_features=len(gen_cols + pca_gen_cols + ohe_cols),
num_cell_features=len(cell_cols + pca_cell_cols + ohe_cols),
num_targets=len(target_cols),
hidden_size=hidden_size)
 model.to(DEVICE)
 model.load_state_dict(torch.load(f".. /input/jp-multiheads/FOLD{fold}_SEED{seed}.pth"))
gen_x_test, cell_x_test = test_[gen_cols + pca_gen_cols + ohe_cols].values, test_[cell_cols + pca_cell_cols + ohe_cols].values
testdataset = TestDataset(gen_x_test, cell_x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 print('predict on test...', end='')
 predictions = np.zeros(( len(test_), targets.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 print(' done.
 ')
 return predictions
with timer("FE 1"):
drop_cols=["bin_0"]
df["ord_5a"]=df["ord_5"].str[0]
df["ord_5b"]=df["ord_5"].str[1]
drop_cols.append("ord_5")
 xor_cols = []
nan_cols = []
for col in df.columns:
tmp_null = df.loc[:,col].isnull().sum()
if tmp_null > 0:
print("{} has {} missing values.. Filling".format(col, tmp_null))
nan_cols.append(col)
 if df.loc[:,col].dtype == "O":
df.loc[:,col].fillna("NAN", inplace=True)
 else:
df.loc[:,col].fillna(-1, inplace=True)
 train_vals = set(df.loc[traindex, col].unique())
 test_vals = set(df.loc[testdex, col].unique())
 xor_cat_vals=train_vals ^ test_vals
if xor_cat_vals:
df.loc[df[col].isin(xor_cat_vals), col]="xor"
print("{} has {} xor factors, {} rows".format(col, len(xor_cat_vals),df.loc[df[col] == 'xor',col].shape[0]))
xor_cols.append(col)
 ordinal_cols=['ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5a', 'day', 'month']
X_oh=df[df.columns.difference(ordinal_cols)]
oh1=pd.get_dummies(X_oh, columns=X_oh.columns, drop_first=True, sparse=True)
 ohc1=oh1.sparse.to_coo()<categorify>
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = get_oof_predictions(fold, seed)
 predictions += pred_ / NFOLDS
return predictions
class ThermometerEncoder(TransformerMixin):

 def __init__(self, sort_key=None):
self.sort_key = sort_key
self.value_map_ = None
def fit(self, X, y=None):
self.value_map_ = {val: i for i, val in enumerate(sorted(X.unique() , key=self.sort_key)) }
return self
def transform(self, X, y=None):
values = X.map(self.value_map_)
 possible_values = sorted(self.value_map_.values())
 idx1 = []
idx2 = []
all_indices = np.arange(len(X))
for idx, val in enumerate(possible_values[:-1]):
new_idxs = all_indices[values > val]
idx1.extend(new_idxs)
 idx2.extend(repeat(idx, len(new_idxs)))
 result = scipy.sparse.coo_matrix(( [1] * len(idx1),(idx1, idx2)) , shape=(len(X), len(possible_values)) , dtype="int8")
 return result<define_variables>
def InferenceMultiHeads(SEED):
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 print(predictions.shape)
 return predictions
predictions = InferenceMultiHeads(SEED )
other_classes = ["NAN", 'xor']
with timer("Thermometer Encoder"):
thermos=[]
for col in ordinal_cols:
if col=="ord_1":
sort_key=(other_classes + ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster'] ).index
elif col=="ord_2":
sort_key=(other_classes + ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot'] ).index
elif col in ["ord_3", "ord_4", "ord_5a"]:
sort_key=str
elif col in ["day", "month"]:
sort_key=int
else:
raise ValueError(col)
 enc=ThermometerEncoder(sort_key=sort_key)
 thermos.append(enc.fit_transform(df[col]))<find_best_params>
predictions_clip = predictions
predictions_clip = pd.DataFrame(predictions_clip, columns=target_cols )
LogisticRegression().get_params()<choose_model_class>
oof_multi_heads_all = np.load('.. /input/jp-multiheads/multi_heads_oof.npy' )
scoring = "roc_auc"
model_names = ['stack',
'logistic1',
'logistic2'
]
n_models = len(model_names)
 Logistic_params_1 = {
"C": 0.123,
"penalty":"l2",
"solver": "lbfgs",
"max_iter": 5000
}
Logistic_params_2 = {
"C": 50,
'penalty':"l2",
"solver": "lbfgs",
"max_iter": 5000
}
folds = KFold(n_splits=3, shuffle=True, random_state=1)
 fold_preds = np.zeros([test.shape[0],n_models])
 oof_preds = np.zeros([X.shape[0],n_models])
 results = {}
with timer("Fit Model"):
estimators = [ 
('logistic_regression_2', LogisticRegression(**Logistic_params_1)) , 
('logistic_regression_1', LogisticRegression(**Logistic_params_2))
]
f, ax = plt.subplots(1,3,figsize = [14,5])
 for i,(trn_idx, val_idx)in enumerate(folds.split(X)) :
clf = StackingClassifier(
 estimators=estimators,
final_estimator=LogisticRegression() , 
)
 with timer("Fold {}".format(str(i))):
clf.fit(X[trn_idx], y.loc[trn_idx])
 tmp_pred = clf.predict_proba(X[val_idx])[:,1]
oof_preds[val_idx,0] = tmp_pred
fold_preds[:,0] += clf.predict_proba(test)[:,1] / folds.n_splits
estimator_performance = {}
estimator_performance['stack_score'] = metrics.roc_auc_score(y.loc[val_idx], tmp_pred)
 for ii, est in enumerate(estimators):
model = clf.named_estimators_[est[0]]
plot_roc_curve(model, X[val_idx], y.loc[val_idx], ax=ax[i])
 pred = model.predict_proba(X[val_idx])[:,1]
oof_preds[val_idx, ii+1] = pred
fold_preds[:,ii+1] += model.predict_proba(test)[:,1] / folds.n_splits
estimator_performance[est[0]+"_score"] = metrics.roc_auc_score(y.loc[val_idx], pred)
 stack_coefficients = {x+"_coefficient":y for(x,y)in zip([x[0] for x in estimators], clf.final_estimator_.coef_[0])}
stack_coefficients['intercept'] = clf.final_estimator_.intercept_[0]
results["Fold {}".format(str(i+1)) ] = [
estimator_performance,
{est[0]+"_iterations":clf.named_estimators_[est[0]].n_iter_ for est in estimators},
stack_coefficients
]
plot_roc_curve(clf, X[val_idx], y.loc[val_idx], ax=ax[i])
 ax[i].plot([0.0, 1.0])
 ax[i].set_title("Fold {} - ROC AUC".format(str(i + 1)))
 plt.tight_layout(pad=2)
 plt.show()
f, ax = plt.subplots(1,2,figsize = [11,5])
 sns.heatmap(pd.DataFrame(oof_preds, columns = model_names ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="magma",ax=ax[0])
 ax[0].set_title("OOF PRED - Correlation Plot")
 sns.heatmap(pd.DataFrame(fold_preds, columns = model_names ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="inferno",ax=ax[1])
 ax[1].set_title("TEST PRED - Correlation Plot")
 plt.tight_layout(pad=3)
 plt.show()<create_dataframe>
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sig_id = test[test['cp_type']!='ctl_vehicle'].sig_id.reset_index(drop=True)
 predictions_clip['sig_id'] = sig_id
sub_mh = pd.merge(test[['sig_id']], predictions_clip, on='sig_id', how='left')
 sub_mh.fillna(0, inplace=True)
 sub_mh.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_mh.to_csv('submission_mh.csv', index=False )
results_pd = pd.DataFrame(results ).T.reset_index()
results_pd.columns = ['Fold','Score','EarlyStopping', 'Coefficients']
results_pd = pd.concat(
 [pd.io.json.json_normalize(results_pd['Score'] ).reset_index(drop=True),
pd.io.json.json_normalize(results_pd['EarlyStopping'] ).reset_index(drop=True),
pd.io.json.json_normalize(results_pd['Coefficients'] ).reset_index(drop=True),
results_pd.reset_index(drop=True)
 ], axis = 1)
 display(results_pd )<save_to_csv>
%who DataFrame
del folds, predictions_clip,targets, test,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols
gc.collect()
%who DataFrame
with timer("Submission"):
pd.DataFrame({'id': testdex, 'target': fold_preds[:,0]} ).to_csv('logistic_stacked_oof_submission.csv', index=False)
 pd.DataFrame({'id': testdex, 'target': fold_preds[:,1]} ).to_csv(estimators[0][0] + '_oof_submission.csv', index=False)
 pd.DataFrame({'id': testdex, 'target': fold_preds[:,2]} ).to_csv(estimators[1][0] + '_oof_submission.csv', index=False )<train_model>
def log_loss_numpy(y_pred):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()
def func_numpy_metric(weights):
oof_blend = np.tensordot(weights, oof, axes =(( 0),(0)))
 return log_loss_numpy(oof_blend)
 def grad_func(weights):
oof_clip = np.clip(oof, 1e-15, 1 - 1e-15)
 gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
@njit
def grad_func_jit(weights):
oof_clip = np.minimum(1 - 1e-15, np.maximum(oof, 1e-15))
gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
print("Notebook Runtime: %0.2f Hours"%(( time.time() - notebookstart)/60/60))<load_from_csv>
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values
oof_dict = {'NN-Transfer-Learning': '.. /input/nn-transferlearning-7seeds4248-10folds/oof_nn_transfer_all.npy',
'TabNet': '.. /input/tabnetfe7seeds424810folds/oof_TabNet_all.npy',
'ResNet': '.. /input/resent-1841/resnet_oof.npy',
'Multi-Heads': '.. /input/jp-multiheads/multi_heads_oof.npy'
}
sub = {}
sub['NN-Transfer-Learning']= sub_nn_transfer
sub['TabNet'] = submission_TabNet
sub['ResNet'] = sub_rs
sub['Multi-Heads'] = sub_mh
oof = []
sub_all = []
oof_tmp = np.zeros([y_true.shape[0], y_true.shape[1]])
 for i in oof_dict:
sub_tmp = sub[i].iloc[:,1:].values
if sub_tmp.sum() > 0.0000000001:
oof.append(np.load(oof_dict[i]))
sub_all.append(sub_tmp)
 oof = np.asarray(oof)
 sub_all = np.asarray(sub_all)
with timer("Load"):
nrow = None
PATH = "/kaggle/input/cat-in-the-dat-ii/"
train = pd.read_csv(PATH + "train.csv", index_col = 'id', nrows = nrow)
 test = pd.read_csv(PATH + "test.csv", index_col = 'id')
 submission_df = pd.read_csv(PATH + "sample_submission.csv")
 [print(x.shape)for x in [train, test, submission_df]]
traindex = train.index
testdex = test.index
y = train.target.copy()
df = pd.concat([train.drop('target',axis = 1), test], axis = 0)
 del train, test, submission_df<categorify>
%%time
log_loss_scores = {}
for n, key in enumerate(oof_dict.keys()):
score_oof = log_loss_numpy(oof[n])
 log_loss_scores[key] = score_oof
print(f'{key} CV:\t', score_oof)
 print('-' * 50 )
with timer("Categorical Processing"):
categorical = df.columns
for col in categorical:
diff = list(set(df.loc[testdex, col].unique())- set(df.loc[traindex,col].unique()))
if diff:
print("Column {} has {} unseen categories in test set".format(col, len(diff)))
 df.loc[df[col].isin(diff),col] = 999
if df[col].dtypes == object:
df[col] = df[col].astype(str)
 lbl = preprocessing.LabelEncoder()
df[col] = pd.Series(lbl.fit_transform(df[col].values)).astype('category' )<groupby>
test_weights = np.array([1 / oof.shape[0]] * oof.shape[0] )
categorical_index = list(range(0, len(categorical)))
 features_names = df.columns
catboost_pool = catboost.Pool(df.loc[traindex,:],
label=y,
cat_features=categorical_index)
 test_pool = catboost.Pool(data=df.loc[testdex,:],
cat_features = categorical_index)
 del df
gc.collect()<init_hyperparams>
tol = 1e-10
init_guess = [1 / oof.shape[0]] * oof.shape[0]
bnds = [(0, 1)for _ in range(oof.shape[0])]
cons = {'type': 'eq',
'fun': lambda x: np.sum(x)- 1,
'jac': lambda x: [1] * len(x)}
print('Inital Blend OOF:', func_numpy_metric(init_guess))
start_time = time()
res_scipy = minimize(fun = func_numpy_metric,
x0 = init_guess,
method = 'SLSQP',
jac = grad_func_jit,
bounds = bnds,
constraints = cons,
tol = tol)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] Optimised Blend OOF:', res_scipy.fun)
 print('Optimised Weights:', res_scipy.x )
def catboost_blackbox(max_depth, reg_lambda):
param = {
'learning_rate': 0.2,
'bagging_temperature': 0.1,
'l2_leaf_reg': reg_lambda,
'depth': int(max_depth),
'iterations' : top_boosting_rounds,
'task_type':'GPU',
'loss_function' : "Logloss",
'objective':'Logloss',
'eval_metric' : "AUC",
'bootstrap_type' : 'Bayesian',
'random_seed': seed,
'early_stopping_rounds' : early_stopping_rounds,
'use_best_model': False,
"verbose": False
}
modelstart= time.time()
scores = catboost.cv(catboost_pool,
param,
fold_count = 2,
stratified = True,
shuffle = True,
partition_random_seed = seed,
plot = False 
)
 runtime =(time.time() - modelstart)/60
optimise = scores.loc[scores['test-AUC-mean'].idxmax() ,'test-AUC-mean'] - scores.loc[scores['test-AUC-mean'].idxmax() ,'test-AUC-std']
optimisation_info.append([scores['test-AUC-mean'].idxmax() , optimise, runtime, param, scores['test-AUC-mean'].idxmax() ])
 return optimise<choose_model_class>
print('Check the sum of all weights:', np.sum(res_scipy.x))
if np.sum(res_scipy.x)- 1 <= tol:
print('Great! The sum of all weights equals to 1!')
 else:
print('Manual adjustion is needed to modify the weights.' )
parameter_bounds = {
'reg_lambda':(0.1, 10),
'max_depth':(3,16)
 }
init_points = 2
n_iter = 4
optimisation_info = []
CATBOOST_BO = BayesianOptimization(catboost_blackbox,
parameter_bounds,
random_state=seed )<train_model>
lb_weight = []
with timer("Bayesian Optimisation - {} Iterations".format(init_points + n_iter)) :
with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 CATBOOST_BO.maximize(init_points = init_points,
n_iter = n_iter,
acq = 'ucb',
xi = 0.0,
alpha = 1e-6 )<compute_test_metric>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(0.25 *sub_nn_transfer.iloc[:, 1:].values + \
0.25 * submission_TabNet.iloc[:,1:].values + \
0.25 * sub_rs.iloc[:,1:].values + \
0.25 * sub_mh.iloc[:,1:].values, 0.00005,0.99995)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )
<prepare_output><EOS> 
!rm submission_TabNet.csv