Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<install_modules>
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 15
plt.rcParams['patch.edgecolor'] = 'k'
warnings.filterwarnings('ignore')
%%capture
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print("Train Dataset: Rows, Columns: ", train_df.shape)
 print("Test Dataset: Rows, Columns: ", test_df.shape )
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN<set_options>
submit = test_df[['Id','idhogar']]
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<define_variables>
train_df['dependency'].value_counts(ascending=False )
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.9
decompo = "PCA"
ncompo_genes = 0.9
ncompo_cells = 0.9
encoding = "dummy"
use_categories = True<load_from_csv>
print(train_df.loc[train_df['dependency']=='no',['SQBdependency']]['SQBdependency'].value_counts())
 print(train_df.loc[train_df['dependency']=='yes',['SQBdependency']]['SQBdependency'].value_counts())
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )<filter>
train_df['dependency'] = train_df['dependency'].replace(( 'yes', 'no'),(1, 0))
test_df['dependency'] = test_df['dependency'].replace(( 'yes', 'no'),(1, 0))
train_df['dependency']=train_df['dependency'].astype(float)
 test_df['dependency']=test_df['dependency'].astype(float )
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )<define_variables>
train_df['edjefe'].value_counts()
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]<concatenate>
train_df['edjefa'].value_counts()
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]<normalization>
print(train_df.loc[train_df['edjefe']=='no',['SQBedjefe']]['SQBedjefe'].value_counts())
 print(train_df.loc[train_df['edjefe']=='yes',['SQBedjefe']]['SQBedjefe'].value_counts() )
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass<categorify>
train_df['edjefa'] = train_df['edjefa'].replace(( 'yes', 'no'),(1, 0))
train_df['edjefe'] = train_df['edjefe'].replace(( 'yes', 'no'),(1, 0))
test_df['edjefa'] = test_df['edjefa'].replace(( 'yes', 'no'),(1, 0))
test_df['edjefe'] = test_df['edjefe'].replace(( 'yes', 'no'),(1, 0))
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )<define_variables>
train_df['edjefa']=train_df['edjefa'].astype(float)
 train_df['edjefe']=train_df['edjefe'].astype(float)
 test_df['edjefa']=test_df['edjefa'].astype(float)
 test_df['edjefe']=test_df['edjefe'].astype(float)
categorical_cols = [cc for cc in data_all.columns if "cp_" in cc]
categorical_cols<feature_engineering>
print("Top Training Columns having missing values:")
 missing_df = train_df.isnull().sum().to_frame()
missing_df = missing_df.sort_values(0, ascending = False)
 missing_df.head()
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
relation_cols = []
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1)
 relation_cols.append("g_" + stats)
 relation_cols.append("c_" + stats)
 relation_cols.append("gc_" + stats )<train_model>
train_df.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes,
random_state = seed)
 pca_genes.fit(data_all[GENES])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed)
 pca_cells.fit(data_all[CELLS])
 g_pca = [f"pca_g-{i}" for i in range(pca_genes.n_components_)]
c_pca = [f"pca_c-{i}" for i in range(pca_cells.n_components_)]
pca_genes_df = pd.DataFrame(pca_genes.transform(data_all[GENES]), columns = g_pca)
 pca_cells_df = pd.DataFrame(pca_cells.transform(data_all[CELLS]), columns = c_pca)
 pca_cols = g_pca + c_pca
if use_categories:
cols_to_get = cols_numeric + relation_cols + categorical_cols
else:
cols_to_get = cols_numeric + relation_cols
data_all = pd.concat([pca_genes_df, pca_cells_df, data_all[cols_to_get]], axis = 1)
 else:
pass<normalization>
train_df['v18q1'] = train_df['v18q1'].fillna(0)
 test_df['v18q1'] = test_df['v18q1'].fillna(0 )
GENES_PCA = [g for g in data_all.columns if "pca_g-" in g]
CELLS_PCA = [c for c in data_all.columns if "pca_c-" in c]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["pca_g-" + stats] = getattr(data_all[GENES_PCA], stats )(axis = 1)
 data_all["pca_c-" + stats] = getattr(data_all[CELLS_PCA], stats )(axis = 1)
 data_all["pca_gc-" + stats] = getattr(data_all[GENES_PCA + CELLS_PCA], stats )(axis = 1 )<define_variables>
print(train_df.loc[train_df['rez_esc'].isnull() ]['age'].value_counts().head(6))
print(train_df.loc[train_df['rez_esc'].isnull() ]['instlevel1'].value_counts())
 print(train_df.loc[train_df['rez_esc'].isnull() ]['instlevel2'].value_counts() )
if use_categories:
cat_idx = []
for col in categorical_cols:
cat_idx.append(data_all.columns.get_loc(col))
print(cat_idx)
 else:
cat_idx = []<load_pretrained>
train_df['rez_esc'] = train_df['rez_esc'].fillna(0)
 test_df['rez_esc'] = test_df['rez_esc'].fillna(0)
 train_df.loc[train_df['rez_esc'] > 5, 'rez_esc'] = 5
test_df.loc[test_df['rez_esc'] > 5, 'rez_esc'] = 5
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )<load_pretrained>
print(train_df['v2a1'].isnull().sum())
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )<drop_column>
print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi1'].value_counts())
 print(train_df['tipovivi1'].value_counts())
features_to_drop = ["cp_type"]
try:
data_all.drop(features_to_drop, axis = 1, inplace = True)
 except:
pass
try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )<import_modules>
train_df.loc[(train_df['v2a1'].isnull() & train_df['tipovivi1'] == 1), 'v2a1'] = 0
test_df.loc[(test_df['v2a1'].isnull() & test_df['tipovivi1'] == 1), 'v2a1'] = 0
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits)+ y_true * np.log(logits)
 return np.mean(-aux )<normalization>
print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi1'].value_counts() )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=1e-5):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=1e-5):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi2'].value_counts())
 print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi3'].value_counts())
 print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi4'].value_counts())
 print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi5'].value_counts() )
MAX_EPOCH = 300
tabnet_params = dict(
 n_d = 64,
n_a = 64,
n_steps = 1,
gamma = 1.2,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 3e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = 28749679,
verbose = 10,
cat_idxs = cat_idx 
)
 scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 5
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 976234598, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 30,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = SmoothBCEwLogits() 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(np.clip(1 /(1 + np.exp(-preds_test)) , 1e-5, 1-1e-5))
oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<compute_test_metric>
train_df.loc[(train_df['v2a1'].isnull() & train_df['tipovivi4'] == 1), 'v2a1'] = 0
test_df.loc[(test_df['v2a1'].isnull() & test_df['tipovivi4'] == 1), 'v2a1'] = 0
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )<save_to_csv>
print(train_df.loc[train_df['v2a1'].isnull() ]['Target'].value_counts() )
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()<import_modules>
train_df['v2a1'].fillna(train_df['v2a1'].mean() , inplace = True)
 test_df['v2a1'].fillna(test_df['v2a1'].mean() , inplace = True )
import os
import random
import time
import json
import warnings
import numpy as np
import pandas as pd
from sklearn.utils.class_weight import compute_class_weight
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.base import BaseEstimator
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, QuantileTransformer
from sklearn.metrics import fbeta_score
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import regularizers
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
import matplotlib.pyplot as plt
from xgboost import XGBClassifier<define_variables>
train_df.fillna(train_df.mean() , inplace = True)
 test_df.fillna(test_df.mean() , inplace = True )
sys.path += ['/kaggle/input/iterstrat', '/kaggle/input/tabnet'] 
<define_variables>
best_correlation = best_correlations.index
best_correlation
def set_random_seeds(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 set_random_seeds(43 )<load_from_csv>
d = {'dependency':'dependency, Dependency rate', 'v18q1':'v18q1, number of tablets household owns', 'epared1':'epared1, if walls are bad', 'qmobilephone':'qmobilephone,
'pisocemento':'pisocemento, =1 if predominant material on the floor is cement',
'eviv1':'eviv1, =1 if floor are bad', 'instlevel8':'instlevel8, =1 undergraduate and higher education', 'rooms':'rooms, number of all rooms in the house', 'r4h1':'r4h1, Males younger than 12 years of age',
'v18q': 'v18q, owns a tablet:', 'edjefe':'edjefe, years of education of male head of household', 'SQBedjefe':'SQBedjefe, years of education of male head of household squared',
'etecho3':'etecho3, =1 if roof are good', 'r4m1':'r4m1, Females younger than 12 years of age', 'SQBovercrowding':'SQBovercrowding, overcrowding squared',
'paredblolad':'paredblolad, =1 if predominant material on the outside wall is block or brick', 'SQBmeaned':'SQBmeaned, square of the mean years of education of adults(>=18)in the household',
'pisomoscer':'pisomoscer, "=1 if predominant material on the floor is mosaic, ceramic, terrazo"', 'overcrowding':'overcrowding,
'eviv3':'eviv3, =1 if floor are good', 'SQBescolari' :'SQBescolari, years of schooling squared',
'escolari':'escolari, years of schooling', 'cielorazo':'cielorazo, =1 if the house has ceiling', 'SQBhogar_nin':'SQBhogar_nin, Number of children 0 to 19 in household, squared',
'r4t1':'r4t1, persons younger than 12 years of age', 'hogar_nin':'hogar_nin, Number of children 0 to 19 in household',
'meaneduc':'meaneduc,average years of education for adults(18+)', 'Target':'Target', 'elimbasu5':'elimbasu5, "=1 if rubbish disposal mainly by throwing in river, creek or sea"'}
X_full = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 Y_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 Y_nonscored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 drug = pd.read_csv('/kaggle/input/lish-moa/train_drug.csv')
 X_test_full = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sig_id = X_test_full['sig_id'].values.reshape(( -1, 1))<define_variables>
print(len(train_df.loc[(train_df['SQBmeaned']>900)]))
print(train_df['SQBmeaned'].value_counts(sort = True))
cat_cols = ['cp_type', 'cp_time', 'cp_dose']
numerical_cols = [c for c in X_full.columns if c not in ['sig_id'] + cat_cols]
label_cols = [c for c in Y_scored.columns if c != 'sig_id']
num_features = X_full.shape[1] - 1
num_labels = Y_scored.shape[1] - 1
num_nonscored_labels = Y_nonscored.shape[1] - 1<compute_train_metric>
to_drop = train_df.loc[(train_df['rooms']>9)|(train_df['r4m1']>3)| 
(train_df['r4t1']>5)|(train_df['hogar_nin']>6)| 
(train_df['meaneduc']>25)| 
(train_df['qmobilephone']>8)|(train_df['r4h1']>2)| 
(train_df['SQBedjefe']>300)|(train_df['SQBescolari']>300)| 
(train_df['SQBhogar_nin']>70)|(train_df['SQBovercrowding']>25)| 
(train_df['SQBmeaned']>900)].index
def logloss(Y_true, Y_preds, label_smoothing=0, remove_unpredicted=False):
if remove_unpredicted:
predicted =(Y_preds.sum(axis=1)!= 0)
 Y_true = Y_true.loc[predicted, :]
Y_preds = Y_preds.loc[predicted, :]
Y_true = Y_true.astype(np.float32)
 return tf.reduce_mean(keras.losses.binary_crossentropy(Y_true, Y_preds, label_smoothing=label_smoothing)).numpy()
def metrics(Y_true, Y_preds):
pr = tf.keras.metrics.Precision()
pr.update_state(Y_true, Y_preds)
 re = tf.keras.metrics.Recall()
re.update_state(Y_true, Y_preds)
 pr = pr.result().numpy()
re = re.result().numpy()
f1 = 2 *(pr * re)/(pr + re)
 auc = tf.keras.metrics.AUC(curve='PR', multi_label=True)
 auc.update_state(Y_true, Y_preds)
 return pr, re, f1, auc.result().numpy()<filter>
train_df.drop(to_drop, inplace=True )
def add_outputs(Y, Y_preds, X=None, idx_list=None, col=None):
if X is not None:
if col is None:
Y.loc[X['cp_type'] == 'trt_cp', :] += Y_preds
else:
Y.loc[X['cp_type'] == 'trt_cp', Y.columns[col]] += Y_preds
elif idx_list is not None:
if col is None:
Y.iloc[idx_list, :] += Y_preds
else:
Y.iloc[idx_list, col] += Y_preds
else:
raise Exception("Must provide X or idx_list" )<categorify>
train_df.groupby('Target' ).mean()
def balance_class_weights(Y_true, class_weights=(0.7, 1.6)) :
n_labels = Y_true.shape[1]
weights = np.empty([n_labels, 2])
 for i in range(n_labels):
if class_weights == 'balanced':
total = len(Y_true[:, i])
 pos = np.sum(Y_true[:, i])
 neg = total - pos
weight_for_0 =(1 / neg)*(total)/2.0
weight_for_1 =(1 / pos)*(total)/2.0
weights[i] = [weight_for_0, weight_for_1]
else:
weights[i] = np.array(class_weights)
 return weights
def weighted_binary_crossentropy(weights, label_smoothing=0):
def weighted_loss(Y_true, Y_pred):
bce = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 return K.mean(
(weights[:, 0]**(1 - Y_true)) *(weights[:, 1]**(Y_true)) * bce(Y_true, Y_pred),
axis=-1 
)
 return weighted_loss<set_options>
train_df['v18q+etecho3'] = train_df['v18q']+train_df['etecho3']
print(train_df.corr() ['Target']['v18q'])
 print(train_df.corr() ['Target']['etecho3'])
 print(train_df.corr() ['Target']['v18q+etecho3'])
 test_df['v18q+etecho3'] = test_df['v18q']+test_df['etecho3']
def clear_session() :
curr_session = tf.compat.v1.get_default_session()
if curr_session is not None:
curr_session.close()
K.clear_session()
s = tf.compat.v1.InteractiveSession()
tf.compat.v1.keras.backend.set_session(s )<categorify>
train_df['v18q+paredblolad'] = train_df['v18q']+train_df['paredblolad']
print(train_df.corr() ['Target']['v18q'])
 print(train_df.corr() ['Target']['paredblolad'])
 print(train_df.corr() ['Target']['v18q+paredblolad'])
 test_df['v18q+paredblolad'] = test_df['v18q']+test_df['paredblolad']
def preprocess(df, targets=None):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df = df.drop(columns='sig_id')
 if targets is not None:
targets = targets.copy()
targets = targets.drop(columns='sig_id')
 targets = targets[df['cp_type'] == 0].reset_index(drop=True)
 df = df[df['cp_type'] == 0].reset_index(drop=True)
 if targets is None:
return df
return df, targets<categorify>
train_df['v18q+pisomoscer'] = train_df['v18q']+train_df['pisomoscer']
print(train_df.corr() ['Target']['v18q'])
 print(train_df.corr() ['Target']['pisomoscer'])
 print(train_df.corr() ['Target']['v18q+pisomoscer'])
 test_df['v18q+pisomoscer'] = test_df['v18q']+test_df['pisomoscer']
def complete_train_labels(Y, train_idx, test_idx):

 train_idx, test_idx = list(train_idx), list(test_idx)
 Y_train = Y.iloc[train_idx, :]
train_pos_cnts = Y_train.sum(axis=0)
 missing_labels = train_pos_cnts[train_pos_cnts == 0].index
removed_from_test = []
for l in missing_labels:
for i in test_idx:
if Y[l].iloc[i] == 1:
train_idx.append(i)
 removed_from_test.append(i)
 break
test_idx = [i for i in test_idx if i not in removed_from_test]
return train_idx, test_idx<define_variables>
train_df['pisomoscer+instlevel8'] = train_df['pisomoscer']+train_df['instlevel8']
print(train_df.corr() ['Target']['pisomoscer'])
 print(train_df.corr() ['Target']['instlevel8'])
 print(train_df.corr() ['Target']['pisomoscer+instlevel8'])
 test_df['pisomoscer+instlevel8'] = test_df['pisomoscer']+test_df['instlevel8']
X_train, Y_train = preprocess(X_full, Y_scored)
 X_test = preprocess(X_test_full)
 Y_train_nonscored = Y_nonscored.drop(columns='sig_id')[X_full['cp_type'] == 'trt_cp'].reset_index(drop=True)
 initial_bias = -np.log(Y_train.mean(axis=0 ).values)
 nonscored_initial_bias = -np.log(Y_train_nonscored.mean(axis=0 ).values + 1e-6 )<merge>
train_df['edjefe+escolari'] = train_df['edjefe']+train_df['escolari']
print(train_df.corr() ['Target']['edjefe'])
 print(train_df.corr() ['Target']['escolari'])
 print(train_df.corr() ['Target']['edjefe+escolari'])
 test_df['edjefe+escolari'] = test_df['edjefe']+test_df['escolari']
def multilabel_split_by_drugs(scored, n_folds, seed):
scored = scored.copy().merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc <= 18].index
vc2 = vc.loc[vc > 18].index
dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)
 tmp = scored.groupby('drug_id')[label_cols].mean().loc[vc1]
for fold,(train_idx, test_idx)in enumerate(skf.split(tmp, tmp[label_cols])) :
dd = {k:fold for k in tmp.index[test_idx].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(train_idx, test_idx)in enumerate(skf.split(tmp, tmp[label_cols])) :
dd = {k:fold for k in tmp.sig_id[test_idx].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() , 'fold'] = scored.loc[scored.fold.isna() , 'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 test_idx = [np.where(scored['fold'] == fold)[0].tolist() for fold in range(n_folds)]
train_idx = [np.where(scored['fold'] != fold)[0].tolist() for fold in range(n_folds)]
return zip(train_idx, test_idx )<feature_engineering>
pip install plotly chart-studio
cufflinks.go_offline(connected=True)
 init_notebook_mode(connected=True )
def select_features(X):
return X.iloc[:, [
1, 2, 3, 4, 5, 6, 7, 9, 11, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31,
32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46,
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 58, 59, 60,
61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88,
89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101,
102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 120, 121, 122, 123, 124, 125, 126, 127, 128,
129, 130, 131, 132, 133, 136, 137, 138, 139, 140, 141, 142, 143,
144, 145, 146, 147, 149, 150, 151, 152, 153, 154, 155, 156, 157,
158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170,
171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183,
184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 197,
198, 199, 200, 202, 203, 204, 205, 206, 208, 209, 210, 211, 212,
213, 214, 215, 216, 217, 218, 219, 220, 221, 223, 224, 225, 226,
227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253,
254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266,
267, 268, 269, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280,
281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294,
295, 296, 298, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309,
310, 311, 312, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336,
337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 367, 368, 369, 370, 371, 374, 375, 376, 377,
378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 390, 391,
392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404,
405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 416, 417, 418,
419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431,
432, 434, 435, 436, 437, 438, 439, 440, 442, 443, 444, 445, 446,
447, 448, 449, 450, 453, 454, 456, 457, 458, 459, 460, 461, 462,
463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475,
476, 477, 478, 479, 481, 482, 483, 484, 485, 486, 487, 488, 489,
490, 491, 492, 493, 494, 495, 496, 498, 500, 501, 502, 503, 505,
506, 507, 509, 510, 511, 512, 513, 514, 515, 518, 519, 520, 521,
522, 523, 524, 525, 526, 527, 528, 530, 531, 532, 534, 535, 536,
538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 549, 550, 551,
552, 554, 557, 559, 560, 561, 562, 565, 566, 567, 568, 569, 570,
571, 572, 573, 574, 575, 577, 578, 580, 581, 582, 583, 584, 585,
586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 599,
600, 601, 602, 606, 607, 608, 609, 611, 612, 613, 615, 616, 617,
618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630,
631, 632, 633, 634, 635, 636, 637, 638, 639, 641, 642, 643, 644,
645, 646, 647, 648, 649, 650, 651, 652, 654, 655, 656, 658, 659,
660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672,
673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685,
686, 687, 688, 689, 691, 692, 693, 694, 695, 696, 697, 699, 700,
701, 702, 704, 705, 707, 708, 709, 710, 711, 713, 714, 716, 717,
718, 720, 721, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732,
733, 734, 735, 737, 738, 739, 740, 742, 743, 744, 745, 746, 747,
748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 759, 760, 761,
762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774,
775, 776, 777, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788,
789, 790, 792, 793, 794, 795, 796, 797, 798, 800, 801, 802, 803,
804, 805, 806, 808, 809, 811, 813, 814, 815, 816, 817, 818, 819,
821, 822, 823, 825, 826, 827, 828, 829, 830, 831, 832, 834, 835,
837, 838, 839, 840, 841, 842, 845, 846, 847, 848, 850, 851, 852,
854, 855, 856, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869,
870, 871, 872, 873, 874
]].values<define_variables>
train_df['Target1'] = train_df.Target
train_df.Target1 = train_df.Target1.apply(str)
 train_df.iplot(
 x='hogar_nin',
y='meaneduc',
categories="Target1",
xTitle = d.get('hogar_nin'),
yTitle = d.get('meaneduc'),
title="Number of children vs.Average education by Poverty level")
 train_df = train_df.drop(['Target1'], axis =1 )
gene_cols = [c for c in X_train.columns if c.startswith('g-')]
cell_cols = [c for c in X_train.columns if c.startswith('c-')]<normalization>
c1 = corrs.abs().unstack().drop_duplicates()
c1.sort_values(ascending = True )
qtransform = joblib.load('/kaggle/input/qtransform/qtransform.joblib')
 X_train_selected = select_features(X_train)
 X_test_selected = select_features(X_test)
 X_train_qtrans = np.concatenate([X_train_selected[:, :2], qtransform.transform(X_train_selected[:, 2:])], axis=1)
 X_test_qtrans = np.concatenate([X_test_selected[:, :2], qtransform.transform(X_test_selected[:, 2:])], axis=1)
 pca_qtrans_2 = joblib.load('/kaggle/input/qtransform/pca_qtrans.joblib' )<load_pretrained>
train_df["Poor_materials"]=train_df['pareddes']+train_df['paredfibras']+train_df['pisonatur']+train_df['pisonotiene']+train_df['techocane']+train_df['epared1']+train_df['etecho1']+train_df['eviv1']
test_df["Poor_materials"]=test_df['pareddes']+test_df['paredfibras']+test_df['pisonatur']+test_df['pisonotiene']+test_df['techocane']+test_df['epared1']+test_df['etecho1']+test_df['eviv1']
print('Pearson correlation coefficients:')
 print('Poor Materials(training set): ',train_df['Poor_materials'].corr(train_df['Target']))
std_scaler_2 = joblib.load('/kaggle/input/qtransform/std_scaler_2.joblib')
 gene_pca_2 = joblib.load('/kaggle/input/qtransform/gene_pca_2.joblib')
 cell_pca_2 = joblib.load('/kaggle/input/qtransform/cell_pca_2.joblib' )<load_pretrained>
train_df["Rich_Materials"]=train_df['paredblolad']+train_df['pisomoscer']+train_df['techoentrepiso']+train_df['techootro']+train_df['cielorazo']+train_df['epared3']+train_df['etecho3']+train_df['eviv3']
test_df["Rich_Materials"]=test_df['paredblolad']+test_df['pisomoscer']+test_df['techoentrepiso']+test_df['techootro']+test_df['cielorazo']+test_df['epared3']+test_df['etecho3']+test_df['eviv3']
print('Pearson correlation coefficients:')
 print('Materials(training set): ',train_df['Rich_Materials'].corr(train_df['Target']))
kmeans_g = joblib.load('/kaggle/input/qtransform/kmeans_g.joblib')
 kmeans_c = joblib.load('/kaggle/input/qtransform/kmeans_c.joblib' )<define_variables>
train_df["Poor_Infrastructure"]=train_df['abastaguano']+train_df['noelec']+train_df['epared1']+train_df['etecho1']+train_df['eviv1']+train_df['lugar3']+train_df['sanitario1']+train_df['energcocinar1']+train_df['elimbasu3']
test_df["Poor_Infrastructure"]=test_df['abastaguano']+test_df['noelec']+test_df['epared1']+test_df['etecho1']+test_df['eviv1']+test_df['lugar3']+test_df['sanitario1']+test_df['energcocinar1']+test_df['elimbasu3']
print('Pearson correlation coefficients:')
 print('Materials(training set): ',train_df['Poor_Infrastructure'].corr(train_df['Target']))
LOAD_PRETRAINED_NN = True
REUSE_SPLITS_NN = True<normalization>
train_df["Good_Infrastructure"]=train_df['sanitario2']+train_df['energcocinar2']+train_df['elimbasu1']+train_df['abastaguadentro']+train_df['planpri']+train_df['epared3']+train_df['etecho3']*(3)+train_df['eviv3']+train_df['lugar1']+train_df['lugar2']+train_df['lugar6']
test_df["Good_Infrastructure"]=test_df['sanitario2']+test_df['energcocinar2']+test_df['elimbasu1']+test_df['abastaguadentro']+test_df['planpri']+test_df['epared3']+test_df['etecho3']*(3)+test_df['eviv3']+test_df['lugar1']+test_df['lugar2']+test_df['lugar6']
print('Pearson correlation coefficients:')
 print('Infrastructure(training set): ',train_df['Good_Infrastructure'].corr(train_df['Target']))
def preprocess_nn(X):
X_pca = pca_qtrans_2.transform(X)
 return np.concatenate([X, X_pca], axis=1 )<categorify>
train_df["overcrowding_total"] = train_df["hacdor"]+train_df["r4t1"] +train_df["instlevel1"] + train_df["area2"]
test_df["overcrowding_total"] = test_df["hacdor"]+ test_df["r4t1"] + test_df["instlevel1"] + test_df["area2"]
print('overcrowding_total: ',train_df['overcrowding_total'].corr(train_df['Target']))
X_train_nn = preprocess_nn(X_train_qtrans)
 X_test_nn = preprocess_nn(X_test_qtrans )<choose_model_class>
train_df["escolari+hacapo"] = train_df["escolari"]+train_df["hacapo"]
test_df["escolari+hacapo"] = test_df["escolari"]+test_df["hacapo"]
print(train_df['escolari+hacapo'].corr(train_df['Target']))
class MultiLabelNN(BaseEstimator):
def __init__(self,
n_input_features=num_features,
n_output_labels=num_labels,
n_hidden_units=2048):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.n_hidden_units = n_hidden_units
self._model_nn = None
def _create_nn_model(self, loss_fn=None, loss_fn_aux=None):
output_bias = keras.initializers.Constant(initial_bias)
 aux_output_bias = keras.initializers.Constant(nonscored_initial_bias)
 inp = keras.layers.Input(self.n_input_features)
 gauss_noise = keras.layers.GaussianNoise(0.001 )(inp)
 norm_1 = keras.layers.BatchNormalization()(gauss_noise)
 dense_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(norm_1)
 norm_2 = keras.layers.BatchNormalization()(dense_1)
 dropout_1 = keras.layers.Dropout(0.4 )(norm_2)
 dense_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_1)
 norm_3 = keras.layers.BatchNormalization()(dense_2)
 dropout_2 = keras.layers.Dropout(0.4 )(norm_3)
 dropout_3 = keras.layers.Dropout(0.5 )(norm_2)
 dense_3 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='relu',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_3)
 norm_4 = keras.layers.BatchNormalization()(dense_3)
 dropout_4 = keras.layers.Dropout(0.4 )(norm_4)
 output_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
), name='main' )(dropout_4)
 output_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 Y_train_nonscored.shape[1],
activation='sigmoid',
bias_initializer=aux_output_bias 
), name='aux' )(dropout_2)
 model_nn = keras.models.Model(inputs=inp, outputs=[output_1, output_2])
 model_nn.compile(
 loss=[
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn is None else loss_fn,
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn_aux is None else loss_fn_aux
],
optimizer=keras.optimizers.Nadam(learning_rate=0.01),
metrics=['binary_crossentropy'],
loss_weights=[0.9, 0.1] 
)
 return model_nn
def fit(self, X, Y, X_valid, Y_valid, Y_aux, Y_aux_valid, max_epochs=100, save_weights=True):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 Y_aux = tf.cast(Y_aux, tf.float32)
 Y_aux_valid = tf.cast(Y_aux_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.7, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 class_weights_aux = balance_class_weights(Y_aux.numpy() , class_weights=(0.7, 1.4))
loss_fn_aux = weighted_binary_crossentropy(class_weights_aux, label_smoothing=1e-3)
 self._model_nn = self._create_nn_model(loss_fn=loss_fn, loss_fn_aux=loss_fn_aux)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=10,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_nn/weights_{repeat}_{fold}.h5',
monitor='val_main_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self._model_nn.fit(
 X, [Y, Y_aux],
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, [Y_valid, Y_aux_valid]),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self._model_nn.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)[0]
return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self._model_nn.save_weights('model_nn/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self._model_nn is None:
self._model_nn = self._create_nn_model()
if pretrained:
base_path = '/kaggle/input/model-nn/'
else:
base_path = './working/'
self._model_nn.load_weights(base_path + 'model_nn/weights_{}_{}.h5'.format(repeat, fold))<define_variables>
pip install --upgrade https://github.com/featuretools/featuretools/zipball/master
n_repeats = 5
n_folds = 7
kfold_seeds = [74, 1156, 415, 608, 1993] if REUSE_SPLITS_NN else np.random.randint(42, 1337, n_repeats)
 kfold_seeds = kfold_seeds[:n_repeats]
assert len(kfold_seeds)== n_repeats
print(kfold_seeds )<split>
ind_bool = list()
ind_ordered = list()
ind_cont = list()
hh_bool = list()
hh_ordered = list()
hh_cont = list()
print(train_df.drop(['age', 'SQBescolari','SQBage', 'agesq'], axis = 1 ).columns.get_loc('Target'))
val_losses = []
histories = []
Y_train_preds_nn = pd.DataFrame(np.zeros(( X_train.shape[0], num_labels)) , columns=label_cols)
 Y_test_preds_nn = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 set_random_seeds(42)
 for repeat, kf_seed in enumerate(kfold_seeds):
kf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=kf_seed, shuffle=True)
 for fold,(train_idx, test_idx)in enumerate(kf.split(X_train, Y_train)) :
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
train_idx, test_idx = complete_train_labels(Y_train, train_idx, test_idx)
 train_idx = tf.random.shuffle(train_idx)
 X_train_fold = tf.gather(X_train_nn, train_idx, axis=0)
 Y_train_fold = tf.gather(Y_train, train_idx, axis=0)
 Y_train_aux_fold = tf.gather(Y_train_nonscored, train_idx, axis=0)
 X_valid_fold = tf.gather(X_train_nn, test_idx, axis=0)
 Y_valid_fold = tf.gather(Y_train, test_idx, axis=0)
 Y_valid_aux_fold = tf.gather(Y_train_nonscored, test_idx, axis=0)
 model_nn = MultiLabelNN(n_input_features=X_train_nn.shape[1])
 if LOAD_PRETRAINED_NN:
model_nn.load_weights(repeat, fold)
 val_loss = model_nn.score(X_valid_fold, Y_valid_fold.numpy())
 else:
history = model_nn.fit(
 X_train_fold, Y_train_fold,
X_valid_fold, Y_valid_fold,
Y_train_aux_fold, Y_valid_aux_fold 
)
 histories.append(history)
 val_loss = min(history.history['val_main_binary_crossentropy'])
 model_nn.load_weights(repeat, fold, pretrained=False)
 val_preds = model_nn.predict(X_valid_fold)[0]
add_outputs(Y_train_preds_nn, val_preds, idx_list=test_idx)
 test_preds = model_nn.predict(X_test_nn)[0]
add_outputs(Y_test_preds_nn, test_preds, X=X_test_full)
 val_losses.append(val_loss)
 print('Val loss: {}'.format(val_loss))
del model_nn
print('----------')
 print('Repeat avg val loss: {}'.format(np.mean(val_losses[repeat*n_folds:(( repeat + 1)*n_folds)])))
 print('Repeat OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_nn /(repeat + 1), remove_unpredicted=True)))
 print('----------')
 Y_train_preds_nn /= n_repeats
Y_test_preds_nn.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats)
 print('==========')
 print('Overall avg val loss: {}'.format(np.mean(val_losses)))
 print('Overall OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_nn, remove_unpredicted=True)) )<define_variables>
test_df['Target'] = np.nan
data = train_df.append(test_df, sort = True)
 for variable in(hh_bool + ind_bool):
data[variable] = data[variable].astype('bool')
 for variable in(hh_cont + ind_cont):
data[variable] = data[variable].astype(float)
 for variable in(hh_ordered + ind_ordered):
try:
data[variable] = data[variable].astype(int)
 except Exception as e:
print(f'Could not convert {variable} because of missing values.' )
LOAD_PRETRAINED_TABNET = True
REUSE_SPLITS_TABNET = True<normalization>
es = ft.EntitySet(id = 'households')
 es.entity_from_dataframe(entity_id = 'data',
dataframe = data,
index = 'Id')

es.normalize_entity(base_entity_id='data',
new_entity_id='household',
index = 'idhogar',
additional_variables = hh_bool + hh_ordered + hh_cont+["Target"] )
def preprocess_tabnet(X, X_qtrans):
X_pca = pca_qtrans_2.transform(X_qtrans)
 g_std = X[gene_cols].std(axis=1 ).values.reshape(-1, 1)
 g_kurt = X[gene_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 g_skew = X[gene_cols].skew(axis=1 ).values.reshape(-1, 1)
 c_std = X[cell_cols].std(axis=1 ).values.reshape(-1, 1)
 c_kurt = X[cell_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 c_skew = X[cell_cols].skew(axis=1 ).values.reshape(-1, 1)
 X_std = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 g_cluster = kmeans_g.predict(X_std[gene_cols] ).reshape(-1, 1)
 c_cluster = kmeans_c.predict(X_std[cell_cols] ).reshape(-1, 1)
 return np.concatenate([
X_qtrans, X_pca,
g_std, g_kurt, g_skew,
c_std, c_kurt, c_skew,
g_cluster, c_cluster
], axis=1 )<concatenate>
feature_matrix, feature_names = ft.dfs(entityset=es,
target_entity = 'household',
max_depth = 2,
verbose = 1,
n_jobs = -1,
chunk_size = 100 )
X_train_tabnet = preprocess_tabnet(X_train, X_train_qtrans)
 X_test_tabnet = preprocess_tabnet(X_test, X_test_qtrans )<choose_model_class>
drop_cols = []
for col in feature_matrix:
if col == 'Target':
pass
else:
if 'Target' in col:
drop_cols.append(col)
 print(drop_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in drop_cols]]
feature_matrix.head()
class MultiLabelTabNet(BaseEstimator):
def __init__(self, n_input_features, n_output_labels):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, loss_fn=None, output_bias=None):
if output_bias is not None:
output_bias = keras.initializers.Constant(output_bias)
 clf = StackedTabNetClassifier(
 feature_columns=None,
num_classes=1024,
num_layers=1,
feature_dim=2048,
output_dim=1024,
num_features=self.n_input_features,
num_decision_steps=1,
relaxation_factor=1.5,
sparsity_coefficient=1e-5,
num_groups=-1,
output_activation='relu' 
)
 model = keras.models.Sequential([
keras.layers.InputLayer(self.n_input_features),
keras.layers.Dropout(0.5),
keras.layers.GaussianNoise(0.001),
clf,
keras.layers.Dropout(0.7),
tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
))
])
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(weight_decay=1e-5),
metrics=['binary_crossentropy'] 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=200):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 15.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(loss_fn=loss_fn, output_bias=initial_bias)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss', min_delta=1e-5,
patience=3, verbose=1, factor=0.1, min_lr=1e-7 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5, patience=10, restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_tabnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=[early_stop_cb, learning_rate_cb, checkpoint_cb],
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_tabnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model()
if pretrained:
base_path = '/kaggle/input/model-tabnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + f'model_tabnet/weights_{repeat}_{fold}.h5' )<define_variables>
train_df = feature_matrix[feature_matrix['Target'].notnull() ].reset_index()
test_df = feature_matrix[feature_matrix['Target'].isnull() ].reset_index()
test_df.head(5 )
n_repeats = 5
n_folds = 7
kfold_seeds = [1217, 231, 902, 3893, 111] if REUSE_SPLITS_TABNET else np.random.randint(42, 1337, n_repeats)
 kfold_seeds = kfold_seeds[:n_repeats]
assert len(kfold_seeds)== n_repeats
print(kfold_seeds )<prepare_x_and_y>
idhogar = test_df['idhogar']
train_df = train_df.select_dtypes(exclude=['object'])
 test_df = test_df.select_dtypes(exclude=['object'])
 train_df = train_df.dropna(axis='columns')
 test_df = test_df.dropna(axis='columns' )
val_losses = []
histories = []
Y_train_preds_tabnet = pd.DataFrame(np.zeros(( X_train.shape[0], num_labels)) , columns=label_cols)
 Y_test_preds_tabnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 set_random_seeds(888)
 for repeat, kf_seed in enumerate(kfold_seeds):
kf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=kf_seed, shuffle=True)
 for fold,(train_idx, test_idx)in enumerate(kf.split(X_train_tabnet, Y_train)) :
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
train_idx, test_idx = complete_train_labels(Y_train, train_idx, test_idx)
 train_idx = tf.random.shuffle(train_idx)
 X_train_fold = tf.gather(X_train_tabnet, train_idx, axis=0)
 Y_train_fold = tf.gather(Y_train, train_idx, axis=0)
 X_valid_fold = tf.gather(X_train_tabnet, test_idx, axis=0)
 Y_valid_fold = tf.gather(Y_train, test_idx, axis=0)
 model_tabnet = MultiLabelTabNet(
 n_input_features=X_train_tabnet.shape[1],
n_output_labels=Y_train.shape[1] 
)
 if LOAD_PRETRAINED_TABNET:
model_tabnet.load_weights(repeat, fold)
 val_loss = model_tabnet.score(X_valid_fold, Y_valid_fold.numpy())
 else:
history = model_tabnet.fit(
 X_train_fold, Y_train_fold,
X_valid_fold, Y_valid_fold,
repeat, fold 
)
 histories.append(history)
 val_loss = min(history.history['val_binary_crossentropy'])
 model_tabnet.load_weights(repeat, fold, pretrained=False)
 val_preds = model_tabnet.predict(X_valid_fold)
 add_outputs(Y_train_preds_tabnet, val_preds, idx_list=test_idx)
 test_preds = model_tabnet.predict(X_test_tabnet)
 add_outputs(Y_test_preds_tabnet, test_preds, X=X_test_full)
 val_losses.append(val_loss)
 print('Val loss: {}'.format(val_loss))
del model_tabnet
print('----------')
 print('Repeat avg val loss: {}'.format(np.mean(val_losses[repeat*n_folds:(( repeat + 1)*n_folds)])))
 print('Repeat OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_tabnet /(repeat + 1), remove_unpredicted=True)))
 print('----------')
 Y_train_preds_tabnet /= n_repeats
Y_test_preds_tabnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats)
 print('==========')
 print('Overall avg val loss: {}'.format(np.mean(val_losses)))
 print('Overall OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_tabnet, remove_unpredicted=True)) )<define_variables>
corr_matrix = train_df.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.99)]
print(f'There are {len(to_drop)} correlated columns to remove.')
 print(to_drop)
 train_df = train_df.drop(columns = to_drop )
LOAD_PRETRAINED_RESNET = True
REUSE_SPLITS_RESNET = True<normalization>
train_df_H20 = train_df
y_df = train_df['Target']
train_df, test_df = train_df.align(test_df, join = 'inner', axis = 1)
 print(f"Training set shape:{train_df.shape}, testing set shape:{test_df.shape}" )
def preprocess_resnet(X):
X_scaled = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 X_genes = gene_pca_2.transform(X_scaled[gene_cols])
 X_cells = cell_pca_2.transform(X_scaled[cell_cols])
 return select_features(X), np.concatenate([X_genes, X_cells], axis=1 )<statistical_test>
X = train_df.values
y = y_df.values
y = y.reshape(-1, 1)
 test_np = test_df.values
X.shape
X_train_resnet_1, X_train_resnet_2 = preprocess_resnet(X_train)
 X_test_resnet_1, X_test_resnet_2 = preprocess_resnet(X_test )<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size = 0.1, random_state = 123)
 X_train.shape
class MultiLabelResNet(BaseEstimator):
def __init__(self, n_input_1, n_input_2, n_output_labels):
self.n_input_1 = n_input_1
self.n_input_2 = n_input_2
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, initial_bias, loss_fn=None):
output_bias = keras.initializers.Constant(initial_bias)
 inp_1 = keras.layers.Input(self.n_input_1, name='raw_inp')
 inp_2 = keras.layers.Input(self.n_input_2, name='pca_inp')
 gauss_2 = keras.layers.GaussianNoise(1e-4 )(inp_2)
 head_1 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(keras.layers.Dense(512, activation='elu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='elu'))
], name='head1')
 seq_1 = head_1(inp_1)
 seq_1_inp_concat = keras.layers.Concatenate()([gauss_2, seq_1])
 head_2 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "elu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "elu"))
], name='head2')
 seq_2 = head_2(seq_1_inp_concat)
 seq_1_seq_2_avg = keras.layers.Average()([seq_1, seq_2])
 head_3 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(206, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(self.n_output_labels, bias_initializer=output_bias, activation='sigmoid'))
], name='head3')
 output = head_3(seq_1_seq_2_avg)
 model = keras.models.Model(inputs=[inp_1, inp_2], outputs=output)
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(learning_rate=0.02, weight_decay=1e-5),
metrics=['binary_crossentropy'], 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=100, save_weights=True, initial_bias=initial_bias, transfer_model=None):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(initial_bias, loss_fn=loss_fn)
 if transfer_model is not None:
self.copy_weights(transfer_model)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=15,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_resnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model(initial_bias)
 if pretrained:
base_path = '/kaggle/input/model-resnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + 'model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def copy_weights(self, from_model):
for to_layer, from_layer in zip(self.estimator_.layers[:-1], from_model.estimator_.layers[:-1]):
to_layer.set_weights(from_layer.get_weights())
 for to_layer, from_layer in zip(self.estimator_.layers[-1].layers[:-1], from_model.estimator_.layers[-1].layers[:-1]):
to_layer.set_weights(from_layer.get_weights() )<define_variables>
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test)
 X = sc.transform(X)
 print(X)
 test_np = sc.transform(test_np)
 print(test_np )
n_repeats = 7
n_folds = 7
kfold_seeds = [188, 526, 1045, 453, 200, 8684, 87352] if REUSE_SPLITS_RESNET else np.random.randint(42, 1337, n_repeats)
 kfold_seeds = kfold_seeds[:n_repeats]
assert len(kfold_seeds)== n_repeats
print(kfold_seeds )<split>
val_losses = []
histories = []
Y_train_preds_resnet = pd.DataFrame(np.zeros(( X_train.shape[0], num_labels)) , columns=label_cols)
 Y_test_preds_resnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 set_random_seeds(123)
 for repeat, kf_seed in enumerate(kfold_seeds):
kf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=kf_seed, shuffle=True)
 for fold,(train_idx, test_idx)in enumerate(kf.split(X_train, Y_train)) :
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
train_idx, test_idx = complete_train_labels(Y_train, train_idx, test_idx)
 train_idx = tf.random.shuffle(train_idx)
 X_train_1_fold = tf.gather(X_train_resnet_1, train_idx, axis=0)
 X_train_2_fold = tf.gather(X_train_resnet_2, train_idx, axis=0)
 Y_train_fold = tf.gather(Y_train, train_idx, axis=0)
 X_valid_1_fold = tf.gather(X_train_resnet_1, test_idx, axis=0)
 X_valid_2_fold = tf.gather(X_train_resnet_2, test_idx, axis=0)
 Y_valid_fold = tf.gather(Y_train, test_idx, axis=0)
 Y_nonscored_train_fold = tf.gather(Y_train_nonscored, train_idx, axis=0)
 Y_nonscored_valid_fold = tf.gather(Y_train_nonscored, test_idx, axis=0)
 model_resnet = MultiLabelResNet(
 n_input_1=X_train_resnet_1.shape[1],
n_input_2=X_train_resnet_2.shape[1],
n_output_labels=Y_train.shape[1] 
)
 if LOAD_PRETRAINED_RESNET:
model_resnet.load_weights(repeat, fold)
 val_loss = model_resnet.score([X_valid_1_fold, X_valid_2_fold], Y_valid_fold.numpy())
 else:
print('Pretrain model')
 transfer_model = MultiLabelResNet(
 n_input_1=X_train_resnet_1.shape[1],
n_input_2=X_train_resnet_2.shape[1],
n_output_labels=Y_train_nonscored.shape[1] 
)
 transfer_model.fit(
 [X_train_1_fold, X_train_2_fold], Y_nonscored_train_fold,
[X_valid_1_fold, X_valid_2_fold], Y_nonscored_valid_fold,
repeat, fold,
max_epochs=10,
save_weights=False,
initial_bias=nonscored_initial_bias 
)
 print('Train model')
 history = model_resnet.fit(
 [X_train_1_fold, X_train_2_fold], Y_train_fold,
[X_valid_1_fold, X_valid_2_fold], Y_valid_fold,
repeat, fold,
transfer_model=transfer_model 
)
 histories.append(history)
 val_loss = min(history.history['val_binary_crossentropy'])
 model_resnet.load_weights(repeat, fold, pretrained=False)
 del transfer_model
val_preds = model_resnet.predict([X_valid_1_fold, X_valid_2_fold])
 add_outputs(Y_train_preds_resnet, val_preds, idx_list=test_idx)
 test_preds = model_resnet.predict([X_test_resnet_1, X_test_resnet_2])
 add_outputs(Y_test_preds_resnet, test_preds, X=X_test_full)
 val_losses.append(val_loss)
 print('Val loss: {}'.format(val_loss))
del model_resnet
print('----------')
 print('Repeat avg val loss: {}'.format(np.mean(val_losses[repeat*n_folds:(( repeat + 1)*n_folds)])))
 print('Repeat OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_resnet /(repeat + 1), remove_unpredicted=True)))
 print('----------')
 Y_train_preds_resnet /= n_repeats
Y_test_preds_resnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats)
 print('==========')
 print('Overall avg val loss: {}'.format(np.mean(val_losses)))
 print('Overall OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_resnet, remove_unpredicted=True)) )<define_variables>
classifier = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.93, min_child_samples = 95, num_leaves = 14, subsample = 0.96 )
<compute_test_metric>
eval_set = [(X_train, y_train),(X_test, y_test)]
classifier.fit(X_train, y_train, eval_metric="multiclass", eval_set=eval_set, verbose=True, early_stopping_rounds=400)
 y_pred = classifier.predict(X_test)
 y_pred = y_pred.reshape(-1, 1 )
weights = [0.22, 0.43, 0.35]
print(sum(weights))
assert(abs(sum(weights)- 1)< 1e-6)
 all_train_preds =(
 Y_train_preds_nn * weights[0] +
Y_train_preds_tabnet * weights[1] +
Y_train_preds_resnet * weights[2] 
)
 print(logloss(Y_train, all_train_preds, remove_unpredicted=True))<feature_engineering>
cm1 = confusion_matrix(y_test, y_pred)
 print(cm1 )
final_preds =(
 Y_test_preds_nn * weights[0] +
Y_test_preds_tabnet * weights[1] +
Y_test_preds_resnet * weights[2] 
)
 final_preds.loc[X_test_full['cp_type'] != 'trt_cp', :] = 0<save_to_csv>
f1 = f1_score(y_test, y_pred, average ='macro')
 print('f1 score for LGBoost model:',f1 )
final_preds = np.concatenate([sig_id, final_preds], axis=1)
 final_preds = pd.DataFrame(final_preds, columns=Y_scored.columns)
 final_preds.to_csv('submission.csv', index=False )<install_modules>
y_pred = classifier.predict(test_np)
 y_pred = y_pred.reshape(-1, 1)
 y_pred = y_pred.astype(int)
 print(plt.hist(y_pred))
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.1-py3-none-any.whl pytorch-tabnet<load_pretrained>
test_df['Target'] = y_pred.astype(int)
 test_df['idhogar'] = idhogar
submit = submit.merge(test_df[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
X, y, genes, cells, classnames, features, X_test, test_control, submission = load()
splits = sorted(glob.glob(".. /input/moa-pipelines/x*.pkl"))
folds, pipes = list(zip(*[joblib.load(split)for split in splits]))
print(splits )<categorify>
submit['Target'] = submit['Target'].fillna(4)
 submit['Target'] = submit['Target'].astype(int )
def tabnet_oof(splits, seeds, pretrained_pipes, tabnet_params, train_params, stop=-1):
S = len(seeds)
 oof_train = np.zeros(( len(X), y.shape[1]), dtype=np.float64)
 oof_test = np.zeros(( len(X_test[~test_control]), y.shape[1]), dtype=np.float64)
 for split,(folds, seed)in enumerate(zip(splits, seeds)) :
nfolds = len(set(folds))
folds_oof = np.zeros(( len(X), y.shape[1]), dtype=np.float64)
 for fold,(t, v)in enumerate(index2folds(folds)) :
pipe = pretrained_pipes[split][fold]
xtrain, xvalid, xtest = X[t].copy() , X[v].copy() , X_test[~test_control].copy()
xtrain = pipe.transform(xtrain)
 xvalid = pipe.transform(xvalid)
 xtest = pipe.transform(xtest)
 model = TabNetRegressor(seed=seed, **tabnet_params)
 model.fit(X_train = xtrain, y_train = y[t].copy() , eval_set = [(xvalid, y[v].copy())], **train_params)
 fold_preds = sigmoid(model.predict(xvalid))
test_preds = sigmoid(model.predict(xtest))
folds_oof[v] += fold_preds
oof_test +=(test_preds / nfolds)
 torch.save(model.network.state_dict() , f"split{split}_seed{seed}_fold{fold}.pth")
 if fold == stop: break
oof_train += folds_oof
oof_train /= S
oof_test /= S
return oof_train, oof_test 
<compute_train_metric>
submit.to_csv('LGBClassification.csv', index = False )
class LogitsLogLoss(Metric):
def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):
return logloss(y_true, sigmoid(y_pred))
def sigmoid(z):
return 1 /(1 + np.exp(-z))
MAX_EPOCH = 200
MASK_TYPE = "entmax"
N = 36
N_STEPS = 1
BATCH_SIZE = 1024
BATCH_SIZE_V = 32
LR = 2e-2
LAMBDA_SPARSE = 0
N_IND = 2
N_SHARED = 2
MOMENTUM = 0.02
PL_FACTOR = 0.9
rdpt_params = dict(mode = "min", patience = 5, min_lr = 1e-5, factor = PL_FACTOR)
 tabnet_params = dict(
 n_d = N,
n_a = N,
n_steps = N_STEPS,
lambda_sparse = LAMBDA_SPARSE,
mask_type = MASK_TYPE,
n_independent = N_IND,
n_shared = N_SHARED,
momentum = MOMENTUM,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr=LR, weight_decay=1e-5),
gamma = 1.3,
scheduler_params = rdpt_params,
scheduler_fn = optim.lr_scheduler.ReduceLROnPlateau,
verbose = 10 
)
 train_params = dict(
 eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
batch_size = BATCH_SIZE,
virtual_batch_size = BATCH_SIZE_V,
patience = 20,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits
)<statistical_test>
print('The prediction was based on LGBoost model with early stopping, trained on ', train_df.shape[1],' features.F1 score for the the training dataset was ',f1,'.' )