Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<compute_test_metric>
!pip install.. /input/pytorch16gpu/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl --quiet
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<train_on_grid>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 cols = [c for c in ss.columns.values if c != 'sig_id']
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train_desc = df_train.Description.fillna("none" ).values
test_desc = df_test.Description.fillna("none" ).values
svd_n_components = 200
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=svd_n_components)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(svd_n_components)])
 df_train = pd.concat(( df_train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(svd_n_components)])
 df_test = pd.concat(( df_test, X_test), axis=1 )<define_variables>
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
y_pred_clip = np.clip(y_pred, 1e-15, 1 - 1e-15)
 return - np.mean(y_true * np.log(y_pred_clip)+(1 - y_true)* np.log(1 - y_pred_clip))
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
del train_targets_nonscored['sig_id']
qt = load('.. /input/moa-preprocess/qt')
 train[GENES+CELLS] = qt.transform(train[GENES+CELLS])
 test[GENES+CELLS] = qt.transform(test[GENES+CELLS])
 n_comp_genes = 600
data = pd.concat([pd.DataFrame(train[GENES]), pd.DataFrame(test[GENES])])
 pca_genes = load('.. /input/moa-preprocess/pca_genes')
 data2 = pca_genes.transform(data[GENES])
 train2 = data2[:train.shape[0]]; test2 = data2[-test.shape[0]:]
train2 = pd.DataFrame(train2, columns = [f'pca_G-{i}' for i in range(n_comp_genes)])
 test2 = pd.DataFrame(test2, columns = [f'pca_G-{i}' for i in range(n_comp_genes)])
 train = pd.concat(( train, train2), axis = 1)
 test = pd.concat(( test, test2), axis = 1)
 n_comp_cells = 50
data = pd.concat([pd.DataFrame(train[CELLS]), pd.DataFrame(test[CELLS])])
 pca_cells = load('.. /input/moa-preprocess/pca_cells')
 data2 = pca_cells.transform(data[CELLS])
 train2 = data2[:train.shape[0]]; test2 = data2[-test.shape[0]:]
train2 = pd.DataFrame(train2, columns = [f'pca_C-{i}' for i in range(n_comp_cells)])
 test2 = pd.DataFrame(test2, columns = [f'pca_C-{i}' for i in range(n_comp_cells)])
 train = pd.concat(( train, train2), axis = 1)
 test = pd.concat(( test, test2), axis = 1)
 data = train.append(test)
 var_thresh = load('.. /input/moa-preprocess/var_thresh')
 data_transformed = var_thresh.transform(data.iloc[:, 3:])
 train_transformed = data_transformed[ : train.shape[0]]
test_transformed = data_transformed[-test.shape[0] : ]
train = pd.DataFrame(train[['cp_type','cp_time','cp_dose']].values.reshape(-1, 3),\
columns=['cp_type','cp_time','cp_dose'])
 train = pd.concat([train, pd.DataFrame(train_transformed)], axis=1)
 test = pd.DataFrame(test[['cp_type','cp_time','cp_dose']].values.reshape(-1, 3),\
columns=['cp_type','cp_time','cp_dose'])
 test = pd.concat([test, pd.DataFrame(test_transformed)], axis=1)
 print(train.shape)
 print(test.shape)
 train_targets = train_targets.loc[train['cp_type'] == 0].reset_index(drop = True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type'] == 0].reset_index(drop = True)
 train = train.loc[train['cp_type'] == 0].reset_index(drop = True)
 print(train.shape)
 top_feats = np.arange(1, train.shape[1])
 print(top_feats)
 cat_tr, cat_test, numerical_tr, numerical_test = train.loc[:, train.columns[1:3]], test.loc[:, test.columns[1:3]], train.loc[:, train.columns[3:]].values, test.loc[:, test.columns[3:]].values
cat_tr.loc[:, 'cp_time'] = cat_tr.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 cat_test.loc[:, 'cp_time'] = cat_test.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 cat_tr = cat_tr.values
cat_test = cat_test.values
targets_tr = train_targets[cols].values.astype(np.float32 )
train_sentiment_path = '.. /input/train_sentiment/'
test_sentiment_path = '.. /input/test_sentiment/'
train_meta_path = '.. /input/train_metadata/'
test_meta_path = '.. /input/test_metadata/'<feature_engineering>
def evals(model, X, y, verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.clamp(y_preds, 0.0,1.0 ).detach().numpy()
score = log_loss_multi(y, y_preds)
 return y_preds, score
def inference_fn(model, X ,verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.sigmoid(torch.as_tensor(y_preds)).numpy()
return y_preds
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
def check_targets(targets):
for i in range(targets.shape[1]):
if len(np.unique(targets[:,i])) != 2:
return False
return True
def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets, self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
sbcewlogits = SmoothBCEwLogits(smoothing = 0.0008 )
df_train[['desc_magnitude', 'desc_score']] = df_train['PetID'].progress_apply(lambda x: get_sentiment(x, train_sentiment_path))
df_test[['desc_magnitude', 'desc_score']] = df_test['PetID'].progress_apply(lambda x: get_sentiment(x, test_sentiment_path))<feature_engineering>
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False, pretrain=False, optimizer_params=None):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 if not pretrain:
self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 else:
self.optimizer = self.optimizer_fn(self.network.parameters() ,
**optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res
<count_missing_values>
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = N_SPLITS
self.EPOCHS = 200
self.num_ensembling = N_STARTS
self.seed = 0
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals = numerical_tr.shape[1]
self.save_name = ".. /input/multilabel-pbestpre-tabnet/tabnet_raw_step1"
self.strategy = "KFOLD"
cfg = Config()
df_train.isna().sum()<define_variables>
X_test = np.concatenate([cat_test, numerical_test], axis=1)
 if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
res = np.zeros(targets_tr.shape)
 for nums, seed in enumerate(range(cfg.num_ensembling)) :
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[ 50,100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 if CALCULATE_OOF:
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 res[val_idx] += preds / cfg.num_ensembling
oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 p = np.stack(p)
 preds_test.append(p)
 if CALCULATE_OOF:
oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
train_petid = df_train['PetID']
test_petid = df_test['PetID']<drop_column>
if CALCULATE_OOF:
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 print(f'TabNet Overall OOF CV Score:', overall_oof_score )
df_train.drop(['Name','Description', 'RescuerID', 'PetID'], axis=1, inplace=True)
 df_test.drop(['Name','Description', 'RescuerID', 'PetID'], axis=1, inplace=True )<compute_test_metric>
submission_tabnet = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_tabnet[cols] = preds_test.mean(1 ).mean(0)
 submission_tabnet.loc[test['cp_type'] == 1, cols] = 0
magnitude_std = df_train.desc_magnitude.std()
magnitude_mean = df_train.desc_magnitude.mean()
score_std = df_train.desc_score.std()
score_mean = df_train.desc_score.mean()
df_train['desc_magnitude'].fillna(np.random.normal(magnitude_mean, magnitude_std), inplace=True)
 df_train['desc_score'].fillna(np.random.normal(score_mean, score_std), inplace=True)
 df_test['desc_magnitude'].fillna(np.random.normal(magnitude_mean, magnitude_std), inplace=True)
 df_test['desc_score'].fillna(np.random.normal(score_mean, score_std), inplace=True )<define_variables>
params = {"batch_size": 128,
"lr": 1e-3,
"weight_decay": 1e-5,
"n_seeds": 3,
"n_folds": 7,
"feat_d": len(top_feats),
"early_stopping_steps": 5,
"hidden_size": 192,
"boost_rate": 1.0,
"num_nets": 40,
"epochs_per_stage": 1,
"correct_epoch": 1,
"model_order": "second"
}
device = "cuda" if torch.cuda.is_available() else "cpu"
<data_type_conversions>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :] , dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
class ForwardType(Enum):
SIMPLE = 0
STACKED = 1
CASCADE = 2
GRADIENT = 3
class DynamicNet(object):
def __init__(self, c0, lr):
self.models = []
self.c0 = c0
self.lr = lr
self.boost_rate = nn.Parameter(torch.tensor(lr, requires_grad=True, device=device))
def add(self, model):
self.models.append(model)
 def parameters(self):
params = []
for m in self.models:
params.extend(m.parameters())
 params.append(self.boost_rate)
 return params
def zero_grad(self):
for m in self.models:
m.zero_grad()
def to_cuda(self):
for m in self.models:
m.to(device)
 def to_eval(self):
for m in self.models:
m.eval()
def to_train(self):
for m in self.models:
m.train(True)
 def forward(self, x):
if len(self.models)== 0:
batch = x.shape[0]
c0 = np.repeat(self.c0.detach().cpu().numpy().reshape(1,-1), batch, axis=0)
 return None, torch.Tensor(c0 ).cuda()
middle_feat_cum = None
prediction = None
with torch.no_grad() :
for m in self.models:
if middle_feat_cum is None:
middle_feat_cum, prediction = m(x, middle_feat_cum)
 else:
middle_feat_cum, pred = m(x, middle_feat_cum)
 prediction += pred
return middle_feat_cum, self.c0 + self.boost_rate * prediction
def forward_grad(self, x):
if len(self.models)== 0:
batch = x.shape[0]
c0 = np.repeat(self.c0.detach().cpu().numpy().reshape(1, -1), batch, axis=0)
 return None, torch.Tensor(c0 ).cuda()
middle_feat_cum = None
prediction = None
for m in self.models:
if middle_feat_cum is None:
middle_feat_cum, prediction = m(x, middle_feat_cum)
 else:
middle_feat_cum, pred = m(x, middle_feat_cum)
 prediction += pred
return middle_feat_cum, self.c0 + self.boost_rate * prediction
@classmethod
def from_file(cls, path, builder):
d = torch.load(path, map_location = torch.device(device))
net = DynamicNet(d['c0'], d['lr'])
 net.boost_rate = d['boost_rate']
for stage, m in enumerate(d['models']):
submod = builder(stage)
 submod.load_state_dict(m)
 net.add(submod)
 return net
def to_file(self, path):
models = [m.state_dict() for m in self.models]
d = {'models': models, 'c0': self.c0, 'lr': self.lr, 'boost_rate': self.boost_rate}
torch.save(d, path)
 class MLP_2HL(nn.Module):
def __init__(self, dim_in, dim_hidden1, dim_hidden2, sparse=False, bn=True):
super(MLP_2HL, self ).__init__()
self.bn2 = nn.BatchNorm1d(dim_in)
 self.layer1 = nn.Sequential(
 nn.utils.weight_norm(nn.Linear(dim_in, dim_hidden1)) ,
nn.LeakyReLU() ,
nn.BatchNorm1d(dim_hidden1),
nn.utils.weight_norm(nn.Linear(dim_hidden1, dim_hidden2)) , 
)
 self.layer2 = nn.Sequential(
 nn.LeakyReLU() ,
nn.utils.weight_norm(nn.Linear(dim_hidden2, 206)) , 
)
 def forward(self, x, lower_f):
if lower_f is not None:
x = torch.cat([x, lower_f], dim=1)
 x = self.bn2(x)
 middle_feat = self.layer1(x)
 out = self.layer2(middle_feat)
 return middle_feat, out
@classmethod
def get_model(cls, stage, params):
if stage == 0:
dim_in = params["feat_d"]
else:
dim_in = params["feat_d"] + params["hidden_size"]
model = MLP_2HL(dim_in, params["hidden_size"], params["hidden_size"])
 return model
def get_optim(params, lr, weight_decay):
optimizer = optim.Adam(params, lr, weight_decay=weight_decay)
 return optimizer
def logloss(net_ensemble, test_loader):
loss = 0
total = 0
loss_f = nn.BCEWithLogitsLoss()
for data in test_loader:
x = data["x"].cuda()
y = data["y"].cuda()
with torch.no_grad() :
_, out = net_ensemble.forward(x)
 loss += loss_f(out, y)
 total += 1
return loss / total
<data_type_conversions>
c0_ = np.log(np.mean(train_targets.values, axis=0))
def inference_fn(seed=0):
seed_everything(seed)
 oof = np.zeros(( train.shape[0], len(cols)))
 overall_score = 0
predictions = np.zeros(( test.shape[0], len(cols)))
 skf = MultilabelStratifiedKFold(n_splits = N_SPLITS, random_state = seed, shuffle = True)
 for foldno,(tr, te)in enumerate(skf.split(train_targets.values, train_targets.values)) :
print(f'Seed {seed}: Fold {foldno}:', end = '\r')
 train_idx = tr
val_idx = te
x_train = train.values[train_idx][:, top_feats]
y_train = train_targets.values[train_idx]
x_val = train.values[val_idx][:, top_feats]
y_val = train_targets.values[val_idx]
train_ds = MoADataset(x_train, y_train)
 val_ds = MoADataset(x_val, y_val)
 train_loader = DataLoader(train_ds, batch_size=params["batch_size"], shuffle=True)
 val_loader = DataLoader(val_ds, batch_size=params["batch_size"], shuffle=False)
 best_score = np.inf
val_score = best_score
best_stage = params["num_nets"] - 1
c0 = torch.tensor(c0_, dtype=torch.float ).to(device)
 net_ensemble = DynamicNet(c0, params["boost_rate"])
 loss_f1 = nn.MSELoss(reduction='none')
 loss_f2 = SmoothBCEwLogits(smoothing=0.0008, reduction="none")
 loss_models = torch.zeros(( params["num_nets"], 3))
all_ensm_losses = []
all_ensm_losses_te = []
all_mdl_losses = []
dynamic_br = []
lr = params["lr"]
L2 = params["weight_decay"]
net_ensemble = DynamicNet.from_file(f".. /input/multilabel-pbestpre-grownet/{foldno}FOLD_{seed}_.pth", lambda stage: MLP_2HL.get_model(stage, params))
net_ensemble.to_cuda()
net_ensemble.to_eval()
if CALCULATE_OOF:
preds = []
with torch.no_grad() :
for data in val_loader:
x = data["x"].to(device)
 _, pred = net_ensemble.forward(x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 oof[val_idx, :] = np.concatenate(preds)
 x_test = test.values[:, top_feats]
test_ds = TestDataset(x_test)
 test_loader = DataLoader(test_ds, batch_size=params["batch_size"], shuffle=False)
 preds = []
with torch.no_grad() :
for data in test_loader:
x = data["x"].to(device)
 _, pred = net_ensemble.forward(x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 predictions += np.concatenate(preds)/ params["n_folds"]
if CALCULATE_OOF:
overall_score = log_loss_metric(train_targets.values, oof)
 print('OOF Score:', overall_score)
 predictions[test['cp_type'] == 1] = 0
sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 sub.loc[:, cols] = predictions
return overall_score, oof, sub
<import_modules>
oof_grownet = np.zeros(( train.shape[0], len(cols)))
 submission_grownet = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_grownet.loc[:, cols] = 0
mean_score = 0
for seed in tqdm(range(N_STARTS)) :
score_seed, oof_seed, sub_seed = inference_fn(seed)
 submission_grownet.loc[:, cols] += sub_seed.loc[:, cols] / N_STARTS
if CALCULATE_OOF:
oof_grownet += oof_seed / N_STARTS
mean_score += score_seed / N_STARTS
from sklearn.model_selection import train_test_split 
<prepare_x_and_y>
if CALCULATE_OOF:
print('Mean OOF Score:', mean_score)
 overall_score_grownet = log_loss_metric(train_targets.values, oof_grownet)
 print('OOF Score:', overall_score_grownet)
 tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all_grownet = np.zeros(tr_targets[cols].shape)
 res_all_grownet[train_features['cp_type'] == 0] = oof_grownet
overall_oof_score_grownet = log_loss_metric(tr_targets[cols].values, res_all_grownet)
 print('Overall OOF CV Score:', overall_oof_score_grownet )
X = df_train.drop(['AdoptionSpeed'], axis=1)
 y = df_train.AdoptionSpeed<split>
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0], name = 'dp0' )(x)
 for i in range(len(hidden_units)) :
x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units[i], name = f'd{i}'), name = f'wn{i}' )(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish, name = f'a{i}' )(x)
 x = tf.keras.layers.BatchNormalization(name = f'bn{i+1}' )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1], name = f'dp{i+1}' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(num_labels,
bias_initializer = tf.keras.initializers.Constant(6.3),
name = f'output_d{num_labels}'),
name = f'output_wn{num_labels}' )(x)
 out = tf.keras.layers.Activation('sigmoid', name = f'output_a{num_labels}' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model
X_train, X_val, y_train, y_val = train_test_split(X, y, train_size=0.8, random_state=1 )<feature_engineering>
def create_rtn(num_columns, num_labels, rethink_iter, num_layers, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0], name = 'dp0' )(x)
 x = tf.keras.layers.RepeatVector(rethink_iter, name = 'rv' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units[0], name = 'd0'), name = 'wn0' )(x)
 x = tf.keras.layers.Activation('swish', name = 'a0' )(x)
 for i in range(num_layers):
if i != num_layers - 1:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = True, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 else:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = False, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(num_labels, name = f'd{num_labels}'), name = f'wn{num_labels}' )(x)
 out = tf.keras.layers.Activation('sigmoid', name = f'out{num_labels}' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model
def train_cats(df):
for n,c in df.items() :
if is_string_dtype(c): df[n] = c.astype('category' ).cat.as_ordered()
def apply_cats(df, trn):
for n,c in df.items() :
if(n in trn.columns)and(trn[n].dtype.name=='category'):
df[n] = pd.Categorical(c, categories=trn[n].cat.categories, ordered=True)
<train_model>
def inference_model(X_train, Y_train_2, Y_nonscored, features, model_name, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name:
model = create_rtn(len(features), 206, **model_params)
 else:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
if CALCULATE_OOF:
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if X_test is not None:
x_tt = X_test.values[:, features]
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 if X_test is not None:
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = log_loss_metric(y_val, val_predict)
 mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub
model_rf = RandomForestClassifier()
model_rf.fit(X_train,y_train)
 model_rf.score(X_train,y_train )<compute_test_metric>
model_names = ['RTN', 'MLP_2', 'MLP_3L_2', 'MLP_4L_0']
model_params = [{'rethink_iter': 2,
'num_layers': 3,
'hidden_units': [256, 384],
'dropout_rates': [0.35975066158109587, 0.5438632643464203],
'lr': 1e-4,
},
{'hidden_units': [128, 896, 1024, 1792],
'dropout_rates': [0.4131287697962003, 0.33921841003415876, 0.13058255266781393, 0.20075775903486198, 0.3354496944535896],
'lr': 1e-4,
},
{'hidden_units': [640, 1536, 256],
'dropout_rates': [0.24204059045909448, 0.5084554762418632, 0.5433377329387503, 0.5644985971088049],
'lr': 1e-4,
},
{'hidden_units': [896, 768, 1280, 128],
'dropout_rates': [0.29522834449539126, 0.47640800252146664, 0.48364063345774644, 0.10344277546242515, 0.49718804108139353],
'lr': 1e-4,
},]
model_rf.score(X_val,y_val )<predict_on_test>
VERBOSE = 0
save_path = '.. /input/multilabel-pbestpre-mlp/'
sample_sub_path = '.. /input/lish-moa/sample_submission.csv'
overall_oof_scores = []
submission_mlp = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 model_name = model_names[m]
oof_score, res, ss = inference_model(train, train_targets, train_targets_nonscored, top_feats, model_name, save_path,
N_STARTS, N_SPLITS, model_params[m], test, sample_sub_path, VERBOSE)
 submission_mlp.append(ss)
 if CALCULATE_OOF:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_scores.append(overall_oof_score)
 print(f'{model_name} Overall OOF CV Score:', overall_oof_score )
print(model_rf.predict(X_train))<compute_train_metric>
if CALCULATE_OOF:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_scores[n] )
def score(model,X,y):
optR = OptimizedRounder()
optR.fit(model.predict(X), y)
 coefficients = optR.coefficients()
pred_y_k = optR.predict(model.predict(X), coefficients)
 print("Valid Counts = ", Counter(y))
print("Predicted Counts = ", Counter(pred_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(y, pred_y_k)
 print("QWK = ", qwk)
 print("RMSE = ",rmse(y, model.predict(X)))
 print("mean Accuracy: ", model.score(X,y)) 
<compute_test_metric>
@njit
def post_process(pred, low, high):
pred_copy = pred.copy()
num_idx = 0
for i in range(pred_copy.shape[0]):
flag = np.zeros(pred_copy.shape[1])
 array = pred_copy[i].copy()
for j in range(pred_copy.shape[1]):
if(pred_copy[i, j] <= low)or(pred_copy[i, j] >= high):
flag[j] = 1
array[j] = round(array[j])
 if flag.all() and pred_copy[i].any() :
pred_copy[i] = array
num_idx += 1
return pred_copy, num_idx
print('Train score: ', score(model_rf,X_train,y_train))<compute_test_metric>
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss[cols] = 0.1296464333222514 * submission_mlp[0][cols].values + \
0.2250012012344317 * submission_mlp[1][cols].values + \
0.06289126408177141 * submission_mlp[2][cols].values + \
0.06845298029278386 * submission_mlp[3][cols].values + \
0.39624897237262097 * submission_tabnet[cols].values + \
0.1177591486961407 * submission_grownet[cols].values
print('valid scoe: ', score(model_rf,X_val,y_val))<train_model>
if POST_PROCESS:
low = 0.015
high = 0.987
ss[cols], num_idx = post_process(ss[cols].values, low, high)
 print(num_idx )
model_rf = RandomForestClassifier(n_estimators=20,n_jobs=-1)
 model_rf.fit(X_train,y_train )<compute_test_metric>
del data, data2, train2, test2, data_transformed, train_transformed, test_transformed
if CALCULATE_OOF:
del tr_targets, overall_oof_score, overall_oof_scores, oof_preds, oof_preds_all, oof_targets, oof_targets_all, oof_score
rubbish = gc.collect()
print('Train score: ', score(model_rf,X_train,y_train))
print('valid scoe: ', score(model_rf,X_val,y_val))<train_model>
if FINETUNE:
pseudo_targets = ss.loc[test['cp_type'] == 0, cols].values
pseudo_train = test.loc[test['cp_type'] == 0, test.columns].values
model_rf = RandomForestClassifier(n_estimators=40,n_jobs=-1)
 model_rf.fit(X_train,y_train)
 print('Train score: ', score(model_rf,X_train,y_train))
print('valid scoe: ', score(model_rf,X_val,y_val))<train_model>
if FINETUNE:
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
res = np.zeros(targets_tr.shape)
 for nums, seed in enumerate(range(cfg.num_ensembling)) :
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 x_tr, y_tr = np.concatenate([cat_tr[train_idx], numerical_tr[train_idx]], axis=1), targets_tr[train_idx]
x_tr, y_tr = np.concatenate([x_tr, X_test[test['cp_type'] == 0, :]]), np.concatenate([y_tr, pseudo_targets])
 X_train, y_train = torch.as_tensor(x_tr), torch.as_tensor(y_tr)
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx]], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 model.fit(X_train=X_train, y_train=y_train, X_valid=X_val, y_valid=y_val, max_epochs=200, patience=5, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=sbcewlogits, pretrain=True, optimizer_params=dict(lr=1e-4, weight_decay=1e-5))
model.load_best_model()
save_name = f"PL_tabnet_raw_step1_fold{j}_{seed}"
model.save_model(save_name)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 if CALCULATE_OOF_PL:
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 res[val_idx] += preds / cfg.num_ensembling
oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 if CALCULATE_OOF_PL:
oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
model_rf = RandomForestClassifier(n_estimators=80,n_jobs=-1)
 model_rf.fit(X_train,y_train)
 print('Train score: ', score(model_rf,X_train,y_train))
print('valid scoe: ', score(model_rf,X_val,y_val))<train_model>
if FINETUNE:
if CALCULATE_OOF_PL:
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 print(f'TabNet Overall OOF CV Score:', overall_oof_score)
 oof_tabnet2 = res_all
model_rf = RandomForestClassifier(n_estimators=100,n_jobs=-1)
 model_rf.fit(X_train,y_train)
 print('Train score: ', score(model_rf,X_train,y_train))
print('valid scoe: ', score(model_rf,X_val,y_val))<train_model>
if FINETUNE:
submission_tabnet2 = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_tabnet2[cols] = preds_test.mean(1 ).mean(0)
 submission_tabnet2.loc[test['cp_type'] == 1, cols] = 0
model_gb = GradientBoostingClassifier(n_estimators=100,)
 model_gb.fit(X_train,y_train)
 print('Train score: ', score(model_gb,X_train,y_train))
print('valid scoe: ', score(model_gb,X_val,y_val))<train_model>
def train_model(X_train, Y_train_2, Y_nonscored, features, model_name, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, pseudo_labeling = True, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name:
model = create_rtn(len(features), 206, **model_params)
 else:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if pseudo_labeling:
x_tr = np.concatenate([x_tr, pseudo_train[:, features]])
 y_tr = np.concatenate([y_tr, pseudo_targets])
 if X_test is not None:
x_tt = X_test.values[:, features]
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 rlr = ReduceLROnPlateau(monitor = 'val_mean_loss', factor = 0.1, patience = 3,
verbose = verbose, min_delta = 1e-4, mode = 'min')
 ckp = ModelCheckpoint(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5', monitor = 'val_mean_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_mean_loss', min_delta = 1e-4, patience = 5, mode = 'min',
baseline = None, restore_best_weights = True, verbose = verbose)
 history = model.fit(x_tr, y_tr, validation_data =(x_val, y_val), epochs = 1000,
batch_size = 128, callbacks = [rlr, ckp, es], verbose = verbose)
 hist = pd.DataFrame(history.history)
 model.load_weights(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5')
 if X_test is not None:
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF_PL:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = hist['val_mean_loss'].min()
mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF_PL:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF_PL:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub
model_gb = GradientBoostingClassifier(n_estimators=150)
 model_gb.fit(X_train,y_train)
 print('Train score: ', score(model_gb,X_train,y_train))
print('valid scoe: ', score(model_gb,X_val,y_val))<train_model>
if FINETUNE:
Pseudo_Labeling = True
VERBOSE = 0
save_path = '.. /input/multilabel-pbestpre-mlp/'
sample_sub_path = '.. /input/lish-moa/sample_submission.csv'
overall_oof_scores = []
oof_mlp2 = []
submission_mlp2 = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 model_name = model_names[m]
oof_score, res, ss = train_model(train, train_targets, train_targets_nonscored, top_feats, model_name, save_path,
N_STARTS, N_SPLITS, model_params[m], test, sample_sub_path, Pseudo_Labeling, VERBOSE)
 submission_mlp2.append(ss)
 if CALCULATE_OOF_PL:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_scores.append(overall_oof_score)
 print(f'{model_name} Overall OOF CV Score:', overall_oof_score)
 oof_mlp2.append(res_all )
model_gb = GradientBoostingClassifier(n_estimators=180)
 model_gb.fit(X,y)
 print('Train score: ', score(model_gb,X,y))
print('Train score: ', score(model_gb,X_train,y_train))
print('valid scoe: ', score(model_gb,X_val,y_val))<train_model>
if FINETUNE:
if CALCULATE_OOF_PL:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_scores[n] )
model_xgb = XGBClassifier(learning_rate=0.09,n_estimators=150,objective='multi:softprob',n_jobs=-1,eval_metric= 'mlogloss',gamma=0.2)
 model_xgb.fit(X_train, y_train)
 print('Train score: ', score(model_xgb,X_train,y_train))
print('valid score: ', score(model_xgb,X_val,y_val))<init_hyperparams>
if FINETUNE:
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss[cols] = 0.1296464333222514 * submission_mlp2[0][cols].values + \
0.2250012012344317 * submission_mlp2[1][cols].values + \
0.06289126408177141 * submission_mlp2[2][cols].values + \
0.06845298029278386 * submission_mlp2[3][cols].values + \
0.39624897237262097 * submission_tabnet2[cols].values + \
0.1177591486961407 * submission_grownet[cols].values
params_lgbm = {'num_leaves': 25,
'min_data_in_leaf': 90,
'objective':'multiclass',
'num_class': 5,
'max_depth': 9,
'learning_rate': 0.03,
"boosting": "gbdt",
"feature_fraction": 0.9980062052116254,
"bagging_freq": 1,
"bagging_fraction": 0.844212672233457,
"bagging_seed": 11,
"metric": 'multi_logloss',
"lambda_l1": 0.12757257166471625,
"random_state": 133,
"verbosity": -1
}<create_dataframe>
ss.to_csv('submission.csv', index = False)
 ss.head(10 )