Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<import_modules>
%matplotlib inline
warnings.filterwarnings("ignore" )
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False, pretrain=False, optimizer_params=None):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 if not pretrain:
self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 else:
self.optimizer = self.optimizer_fn(self.network.parameters() ,
**optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res<set_options>
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = N_SPLITS
self.EPOCHS = 200
self.num_ensembling = N_STARTS
self.seed = 0
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals = numerical_tr.shape[1]
self.save_name = ".. /input/multilabel-pbestpre-tabnet/tabnet_raw_step1"
self.strategy = "KFOLD"
cfg = Config()<define_variables>
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df
X_test = np.concatenate([cat_test, numerical_test], axis=1)
 if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
res = np.zeros(targets_tr.shape)
 for nums, seed in enumerate(range(cfg.num_ensembling)) :
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[ 50,100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 if CALCULATE_OOF:
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 res[val_idx] += preds / cfg.num_ensembling
oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 p = np.stack(p)
 preds_test.append(p)
 if CALCULATE_OOF:
oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )<load_from_csv>
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
if CALCULATE_OOF:
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 print(f'TabNet Overall OOF CV Score:', overall_oof_score )<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id
submission_tabnet = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_tabnet[cols] = preds_test.mean(1 ).mean(0)
 submission_tabnet.loc[test['cp_type'] == 1, cols] = 0<init_hyperparams>
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )
params = {"batch_size": 128,
"lr": 1e-3,
"weight_decay": 1e-5,
"n_seeds": 3,
"n_folds": 7,
"feat_d": len(top_feats),
"early_stopping_steps": 5,
"hidden_size": 192,
"boost_rate": 1.0,
"num_nets": 40,
"epochs_per_stage": 1,
"correct_epoch": 1,
"model_order": "second"
}
device = "cuda" if torch.cuda.is_available() else "cpu"<categorify>
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefe'] == "no", "edjefe"] = 0
train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "edjefa"] = train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "escolari"]
train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "edjefe"] = train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "edjefa"] = test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "edjefe"] = test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "escolari"]
train.loc[train['edjefa'] == "yes", "edjefa"] = 4
train.loc[train['edjefe'] == "yes", "edjefe"] = 4
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
train['edjefe'] = train['edjefe'].astype("int")
 train['edjefa'] = train['edjefa'].astype("int")
 test['edjefe'] = test['edjefe'].astype("int")
 test['edjefa'] = test['edjefa'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :] , dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
class ForwardType(Enum):
SIMPLE = 0
STACKED = 1
CASCADE = 2
GRADIENT = 3
class DynamicNet(object):
def __init__(self, c0, lr):
self.models = []
self.c0 = c0
self.lr = lr
self.boost_rate = nn.Parameter(torch.tensor(lr, requires_grad=True, device=device))
def add(self, model):
self.models.append(model)
 def parameters(self):
params = []
for m in self.models:
params.extend(m.parameters())
 params.append(self.boost_rate)
 return params
def zero_grad(self):
for m in self.models:
m.zero_grad()
def to_cuda(self):
for m in self.models:
m.to(device)
 def to_eval(self):
for m in self.models:
m.eval()
def to_train(self):
for m in self.models:
m.train(True)
 def forward(self, x):
if len(self.models)== 0:
batch = x.shape[0]
c0 = np.repeat(self.c0.detach().cpu().numpy().reshape(1,-1), batch, axis=0)
 return None, torch.Tensor(c0 ).cuda()
middle_feat_cum = None
prediction = None
with torch.no_grad() :
for m in self.models:
if middle_feat_cum is None:
middle_feat_cum, prediction = m(x, middle_feat_cum)
 else:
middle_feat_cum, pred = m(x, middle_feat_cum)
 prediction += pred
return middle_feat_cum, self.c0 + self.boost_rate * prediction
def forward_grad(self, x):
if len(self.models)== 0:
batch = x.shape[0]
c0 = np.repeat(self.c0.detach().cpu().numpy().reshape(1, -1), batch, axis=0)
 return None, torch.Tensor(c0 ).cuda()
middle_feat_cum = None
prediction = None
for m in self.models:
if middle_feat_cum is None:
middle_feat_cum, prediction = m(x, middle_feat_cum)
 else:
middle_feat_cum, pred = m(x, middle_feat_cum)
 prediction += pred
return middle_feat_cum, self.c0 + self.boost_rate * prediction
@classmethod
def from_file(cls, path, builder):
d = torch.load(path, map_location = torch.device(device))
net = DynamicNet(d['c0'], d['lr'])
 net.boost_rate = d['boost_rate']
for stage, m in enumerate(d['models']):
submod = builder(stage)
 submod.load_state_dict(m)
 net.add(submod)
 return net
def to_file(self, path):
models = [m.state_dict() for m in self.models]
d = {'models': models, 'c0': self.c0, 'lr': self.lr, 'boost_rate': self.boost_rate}
torch.save(d, path)
 class MLP_2HL(nn.Module):
def __init__(self, dim_in, dim_hidden1, dim_hidden2, sparse=False, bn=True):
super(MLP_2HL, self ).__init__()
self.bn2 = nn.BatchNorm1d(dim_in)
 self.layer1 = nn.Sequential(
 nn.utils.weight_norm(nn.Linear(dim_in, dim_hidden1)) ,
nn.LeakyReLU() ,
nn.BatchNorm1d(dim_hidden1),
nn.utils.weight_norm(nn.Linear(dim_hidden1, dim_hidden2)) , 
)
 self.layer2 = nn.Sequential(
 nn.LeakyReLU() ,
nn.utils.weight_norm(nn.Linear(dim_hidden2, 206)) , 
)
 def forward(self, x, lower_f):
if lower_f is not None:
x = torch.cat([x, lower_f], dim=1)
 x = self.bn2(x)
 middle_feat = self.layer1(x)
 out = self.layer2(middle_feat)
 return middle_feat, out
@classmethod
def get_model(cls, stage, params):
if stage == 0:
dim_in = params["feat_d"]
else:
dim_in = params["feat_d"] + params["hidden_size"]
model = MLP_2HL(dim_in, params["hidden_size"], params["hidden_size"])
 return model
def get_optim(params, lr, weight_decay):
optimizer = optim.Adam(params, lr, weight_decay=weight_decay)
 return optimizer
def logloss(net_ensemble, test_loader):
loss = 0
total = 0
loss_f = nn.BCEWithLogitsLoss()
for data in test_loader:
x = data["x"].cuda()
y = data["y"].cuda()
with torch.no_grad() :
_, out = net_ensemble.forward(x)
 loss += loss_f(out, y)
 total += 1
return loss / total<init_hyperparams>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
c0_ = np.log(np.mean(train_targets.values, axis=0))
def inference_fn(seed=0):
seed_everything(seed)
 oof = np.zeros(( train.shape[0], len(cols)))
 overall_score = 0
predictions = np.zeros(( test.shape[0], len(cols)))
 skf = MultilabelStratifiedKFold(n_splits = N_SPLITS, random_state = seed, shuffle = True)
 for foldno,(tr, te)in enumerate(skf.split(train_targets.values, train_targets.values)) :
print(f'Seed {seed}: Fold {foldno}:', end = '\r')
 train_idx = tr
val_idx = te
x_train = train.values[train_idx][:, top_feats]
y_train = train_targets.values[train_idx]
x_val = train.values[val_idx][:, top_feats]
y_val = train_targets.values[val_idx]
train_ds = MoADataset(x_train, y_train)
 val_ds = MoADataset(x_val, y_val)
 train_loader = DataLoader(train_ds, batch_size=params["batch_size"], shuffle=True)
 val_loader = DataLoader(val_ds, batch_size=params["batch_size"], shuffle=False)
 best_score = np.inf
val_score = best_score
best_stage = params["num_nets"] - 1
c0 = torch.tensor(c0_, dtype=torch.float ).to(device)
 net_ensemble = DynamicNet(c0, params["boost_rate"])
 loss_f1 = nn.MSELoss(reduction='none')
 loss_f2 = SmoothBCEwLogits(smoothing=0.0008, reduction="none")
 loss_models = torch.zeros(( params["num_nets"], 3))
all_ensm_losses = []
all_ensm_losses_te = []
all_mdl_losses = []
dynamic_br = []
lr = params["lr"]
L2 = params["weight_decay"]
net_ensemble = DynamicNet.from_file(f".. /input/multilabel-pbestpre-grownet/{foldno}FOLD_{seed}_.pth", lambda stage: MLP_2HL.get_model(stage, params))
net_ensemble.to_cuda()
net_ensemble.to_eval()
if CALCULATE_OOF:
preds = []
with torch.no_grad() :
for data in val_loader:
x = data["x"].to(device)
 _, pred = net_ensemble.forward(x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 oof[val_idx, :] = np.concatenate(preds)
 x_test = test.values[:, top_feats]
test_ds = TestDataset(x_test)
 test_loader = DataLoader(test_ds, batch_size=params["batch_size"], shuffle=False)
 preds = []
with torch.no_grad() :
for data in test_loader:
x = data["x"].to(device)
 _, pred = net_ensemble.forward(x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 predictions += np.concatenate(preds)/ params["n_folds"]
if CALCULATE_OOF:
overall_score = log_loss_metric(train_targets.values, oof)
 print('OOF Score:', overall_score)
 predictions[test['cp_type'] == 1] = 0
sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 sub.loc[:, cols] = predictions
return overall_score, oof, sub<load_from_csv>
train, test = train_test_apply_func(train, test, convert_OHE2LE )
oof_grownet = np.zeros(( train.shape[0], len(cols)))
 submission_grownet = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_grownet.loc[:, cols] = 0
mean_score = 0
for seed in tqdm(range(N_STARTS)) :
score_seed, oof_seed, sub_seed = inference_fn(seed)
 submission_grownet.loc[:, cols] += sub_seed.loc[:, cols] / N_STARTS
if CALCULATE_OOF:
oof_grownet += oof_seed / N_STARTS
mean_score += score_seed / N_STARTS<compute_train_metric>
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )
if CALCULATE_OOF:
print('Mean OOF Score:', mean_score)
 overall_score_grownet = log_loss_metric(train_targets.values, oof_grownet)
 print('OOF Score:', overall_score_grownet)
 tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all_grownet = np.zeros(tr_targets[cols].shape)
 res_all_grownet[train_features['cp_type'] == 0] = oof_grownet
overall_oof_score_grownet = log_loss_metric(tr_targets[cols].values, res_all_grownet)
 print('Overall OOF CV Score:', overall_oof_score_grownet )<choose_model_class>
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
df['dependency_yes'] = df['dependency'].apply(lambda x: 1 if x == 'yes' else 0)
 df['dependency_no'] = df['dependency'].apply(lambda x: 1 if x == 'no' else 0)
 extract_features(train)
 extract_features(test )
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0], name = 'dp0' )(x)
 for i in range(len(hidden_units)) :
x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units[i], name = f'd{i}'), name = f'wn{i}' )(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish, name = f'a{i}' )(x)
 x = tf.keras.layers.BatchNormalization(name = f'bn{i+1}' )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1], name = f'dp{i+1}' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(num_labels,
bias_initializer = tf.keras.initializers.Constant(6.3),
name = f'output_d{num_labels}'),
name = f'output_wn{num_labels}' )(x)
 out = tf.keras.layers.Activation('sigmoid', name = f'output_a{num_labels}' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model<choose_model_class>
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )
def create_rtn(num_columns, num_labels, rethink_iter, num_layers, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0], name = 'dp0' )(x)
 x = tf.keras.layers.RepeatVector(rethink_iter, name = 'rv' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units[0], name = 'd0'), name = 'wn0' )(x)
 x = tf.keras.layers.Activation('swish', name = 'a0' )(x)
 for i in range(num_layers):
if i != num_layers - 1:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = True, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 else:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = False, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(num_labels, name = f'd{num_labels}'), name = f'wn{num_labels}' )(x)
 out = tf.keras.layers.Activation('sigmoid', name = f'out{num_labels}' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model<load_from_csv>
def split_data(train, y, households, test_percentage=0.20, seed=None):
np.random.seed(seed=seed)
 train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
return X_train, y_train, X_test, y_test
def inference_model(X_train, Y_train_2, Y_nonscored, features, model_name, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name:
model = create_rtn(len(features), 206, **model_params)
 else:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
if CALCULATE_OOF:
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if X_test is not None:
x_tt = X_test.values[:, features]
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 if X_test is not None:
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = log_loss_metric(y_val, val_predict)
 mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub<init_hyperparams>
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=1977)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.1), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar
model_names = ['RTN', 'MLP_2', 'MLP_3L_2', 'MLP_4L_0']
model_params = [{'rethink_iter': 2,
'num_layers': 3,
'hidden_units': [256, 384],
'dropout_rates': [0.35975066158109587, 0.5438632643464203],
'lr': 1e-4,
},
{'hidden_units': [128, 896, 1024, 1792],
'dropout_rates': [0.4131287697962003, 0.33921841003415876, 0.13058255266781393, 0.20075775903486198, 0.3354496944535896],
'lr': 1e-4,
},
{'hidden_units': [640, 1536, 256],
'dropout_rates': [0.24204059045909448, 0.5084554762418632, 0.5433377329387503, 0.5644985971088049],
'lr': 1e-4,
},
{'hidden_units': [896, 768, 1280, 128],
'dropout_rates': [0.29522834449539126, 0.47640800252146664, 0.48364063345774644, 0.10344277546242515, 0.49718804108139353],
'lr': 1e-4,
},]<define_variables>
extra_drop_features = ['geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'parentesco1',
'rez_esc',
'dependency_yes',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
'dependency_no'] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']
VERBOSE = 0
save_path = '.. /input/multilabel-pbestpre-mlp/'
sample_sub_path = '.. /input/lish-moa/sample_submission.csv'
overall_oof_scores = []
submission_mlp = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 model_name = model_names[m]
oof_score, res, ss = inference_model(train, train_targets, train_targets_nonscored, top_feats, model_name, save_path,
N_STARTS, N_SPLITS, model_params[m], test, sample_sub_path, VERBOSE)
 submission_mlp.append(ss)
 if CALCULATE_OOF:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_scores.append(overall_oof_score)
 print(f'{model_name} Overall OOF CV Score:', overall_oof_score )<compute_test_metric>
lgb_drop_cols = extra_drop_features + ["idhogar"]
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )
if CALCULATE_OOF:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_scores[n] )<categorify>
kf = StratifiedKFold(n_splits=6, shuffle=True)
 folds = []
for train_index, test_index in kf.split(X_train, y_train):
folds.append(( train_index, test_index))
@njit
def post_process(pred, low, high):
pred_copy = pred.copy()
num_idx = 0
for i in range(pred_copy.shape[0]):
flag = np.zeros(pred_copy.shape[1])
 array = pred_copy[i].copy()
for j in range(pred_copy.shape[1]):
if(pred_copy[i, j] <= low)or(pred_copy[i, j] >= high):
flag[j] = 1
array[j] = round(array[j])
 if flag.all() and pred_copy[i].any() :
pred_copy[i] = array
num_idx += 1
return pred_copy, num_idx<load_from_csv>
opt_parameters = {'colsample_bytree': 0.93, 'min_child_samples': 56, 'num_leaves': 19, 'subsample': 0.84}
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss[cols] = 0.1296464333222514 * submission_mlp[0][cols].values + \
0.2250012012344317 * submission_mlp[1][cols].values + \
0.06289126408177141 * submission_mlp[2][cols].values + \
0.06845298029278386 * submission_mlp[3][cols].values + \
0.39624897237262097 * submission_tabnet[cols].values + \
0.1177591486961407 * submission_grownet[cols].values<sort_values>
def _parallel_fit_estimator(estimator, X, y, train_index, test_index, sample_weight=None, **fit_params):
X_train = X.iloc[train_index]
y_train = y[train_index]
X_test = X.iloc[test_index]
y_test = y[test_index]
fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X_train, y_train, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y, train_index, test_index,
sample_weight=sample_weight, **fit_params)
 for(clf,(train_index, test_index)) in product(clfs, folds))
return self
if POST_PROCESS:
low = 0.015
high = 0.987
ss[cols], num_idx = post_process(ss[cols].values, low, high)
 print(num_idx )<drop_column>
clfs = []
for i in range(10):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i*2, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]
del data, data2, train2, test2, data_transformed, train_transformed, test_transformed
if CALCULATE_OOF:
del tr_targets, overall_oof_score, overall_oof_scores, oof_preds, oof_preds_all, oof_targets, oof_targets_all, oof_score
rubbish = gc.collect()<define_variables>
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))
if FINETUNE:
pseudo_targets = ss.loc[test['cp_type'] == 0, cols].values
pseudo_train = test.loc[test['cp_type'] == 0, test.columns].values<choose_model_class>
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))
if FINETUNE:
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
res = np.zeros(targets_tr.shape)
 for nums, seed in enumerate(range(cfg.num_ensembling)) :
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 x_tr, y_tr = np.concatenate([cat_tr[train_idx], numerical_tr[train_idx]], axis=1), targets_tr[train_idx]
x_tr, y_tr = np.concatenate([x_tr, X_test[test['cp_type'] == 0, :]]), np.concatenate([y_tr, pseudo_targets])
 X_train, y_train = torch.as_tensor(x_tr), torch.as_tensor(y_tr)
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx]], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 model.fit(X_train=X_train, y_train=y_train, X_valid=X_val, y_valid=y_val, max_epochs=200, patience=5, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=sbcewlogits, pretrain=True, optimizer_params=dict(lr=1e-4, weight_decay=1e-5))
model.load_best_model()
save_name = f"PL_tabnet_raw_step1_fold{j}_{seed}"
model.save_model(save_name)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 if CALCULATE_OOF_PL:
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 res[val_idx] += preds / cfg.num_ensembling
oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 if CALCULATE_OOF_PL:
oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )<load_from_csv>
ranked_features = feature_importance(clf_final, X_train )
if FINETUNE:
if CALCULATE_OOF_PL:
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 print(f'TabNet Overall OOF CV Score:', overall_oof_score)
 oof_tabnet2 = res_all<load_from_csv>
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids
if FINETUNE:
submission_tabnet2 = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_tabnet2[cols] = preds_test.mean(1 ).mean(0)
 submission_tabnet2.loc[test['cp_type'] == 1, cols] = 0<load_from_csv>
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1
def train_model(X_train, Y_train_2, Y_nonscored, features, model_name, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, pseudo_labeling = True, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name:
model = create_rtn(len(features), 206, **model_params)
 else:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if pseudo_labeling:
x_tr = np.concatenate([x_tr, pseudo_train[:, features]])
 y_tr = np.concatenate([y_tr, pseudo_targets])
 if X_test is not None:
x_tt = X_test.values[:, features]
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 rlr = ReduceLROnPlateau(monitor = 'val_mean_loss', factor = 0.1, patience = 3,
verbose = verbose, min_delta = 1e-4, mode = 'min')
 ckp = ModelCheckpoint(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5', monitor = 'val_mean_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_mean_loss', min_delta = 1e-4, patience = 5, mode = 'min',
baseline = None, restore_best_weights = True, verbose = verbose)
 history = model.fit(x_tr, y_tr, validation_data =(x_val, y_val), epochs = 1000,
batch_size = 128, callbacks = [rlr, ckp, es], verbose = verbose)
 hist = pd.DataFrame(history.history)
 model.load_weights(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5')
 if X_test is not None:
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF_PL:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = hist['val_mean_loss'].min()
mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF_PL:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF_PL:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub<define_variables>
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )