Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<save_to_csv>
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<import_modules>
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
import pickle
import numpy as np
import pandas as pd
import tensorflow as tf<define_variables>
from collections import OrderedDict
MIXED_PRECISION = False
XLA_ACCELERATE = True<set_options>
map={'yes':1,"no":0}
for df in [train,test]:
df['dependency']=df['dependency'].replace(map ).astype(np.float64)
 df['edjefe']=df['edjefe'].replace(map ).astype(np.float64)
 df['edjefa']=df['edjefa'].replace(map ).astype(np.float64)
 train[['dependency','edjefe','edjefa']].describe()
if MIXED_PRECISION:
policy = tf.keras.mixed_precision.experimental.Policy("mixed_float16")
 tf.keras.mixed_precision.experimental.set_policy(policy)
 tf.config.optimizer.set_jit(XLA_ACCELERATE )<choose_model_class>
test['Target']=np.nan
data=train.append(test,ignore_index=True )
def build_callbacks(
 checkpoint_path,
factor=0.1,
mode="auto",
monitor="val_loss",
patience_for_es=0,
patience_for_rlop=10,
save_best_only=False,
verbose=0, 
):
callbacks = []
if checkpoint_path is not None:
model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
 checkpoint_path,
mode=mode,
monitor=monitor,
save_best_only=save_best_only,
verbose=verbose, 
)
 callbacks.append(model_checkpoint)
 if patience_for_es is not None:
early_stopping = tf.keras.callbacks.EarlyStopping(
 mode=mode, monitor=monitor, patience=patience_for_es, verbose=verbose 
)
 callbacks.append(early_stopping)
 if patience_for_rlop is not None:
reduce_lr_on_plateau = tf.keras.callbacks.ReduceLROnPlateau(
 factor=factor,
monitor=monitor,
mode=mode,
patience=patience_for_rlop,
verbose=verbose, 
)
 callbacks.append(reduce_lr_on_plateau)
 return callbacks<choose_model_class>
X = [[0],[1],[2]]
Y = [ 1, 2, 1 ]
dtc = DTC(max_depth=1 )
def build_model(
 input_dim,
output_dim,
activation="relu",
kernel_initializer="glorot_uniform",
label_smoothing=0.0,
momentum=0.99,
n_layers=3,
n_units="auto",
optimizer_params=None,
rate=0.0, 
):
if n_units == "auto":
n_units = 0.5 *(input_dim + output_dim)
 inputs = tf.keras.layers.Input(shape=input_dim)
 x = inputs
for i in range(n_layers - 2):
x = tf.keras.layers.Dense(n_units, kernel_initializer=kernel_initializer )(x)
 x = tf.keras.layers.BatchNormalization(momentum=momentum )(x)
 x = tf.keras.layers.Activation(activation )(x)
 x = tf.keras.layers.Dropout(rate )(x)
 x = tf.keras.layers.Dense(output_dim )(x)
 outputs = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inputs, outputs=outputs)
 if optimizer_params is None:
optimizer_params = {}
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 optimizer = tf.keras.optimizers.Adam(**optimizer_params)
 model.compile(loss=loss, optimizer=optimizer)
 return model<compute_test_metric>
dtc.fit(X,Y)
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
def score(Y, Y_pred, eps=1e-15, label_smoothing=0.0):
Y = np.asarray(Y)
 Y = np.ravel(Y)
 if label_smoothing > 0.0:
Y = Y *(1.0 - label_smoothing)+ 0.5 * label_smoothing
Y_pred = np.asarray(Y_pred)
 Y_pred = np.ravel(Y_pred)
 Y_pred = np.clip(Y_pred, eps, 1.0 - eps)
 return -np.mean(Y * np.log(Y_pred)+(1.0 - Y)* np.log(1.0 - Y_pred))<set_options>
dtc.fit(X,Y,sample_weight=[1,2,3])
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
def set_seed(seed=0):
os.environ["PYTHONHASHSEED"] = str(seed)
 rn.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 graph = tf.compat.v1.get_default_graph()
session_conf = tf.compat.v1.ConfigProto(
 inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=graph, config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess )<features_selection>
data['Target'].value_counts()
class MultilabelStratifiedGroupKFold(_BaseKFold):
def __init__(self, n_splits=5, random_state=None, shuffle=False):
super().__init__(n_splits=n_splits, random_state=random_state, shuffle=shuffle)
 def _iter_test_indices(self, X=None, y=None, groups=None):
cv = MultilabelStratifiedKFold(
 n_splits=self.n_splits,
random_state=self.random_state,
shuffle=self.shuffle, 
)
 value_counts = groups.value_counts()
regluar_indices = value_counts.loc[value_counts <= 18].index.sort_values()
irregluar_indices = value_counts.loc[value_counts > 18].index.sort_values()
group_to_fold = {}
tmp = y.groupby(groups ).mean().loc[regluar_indices]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
group_to_fold.update({group: fold for group in tmp.index[test]})
 sample_to_fold = {}
tmp = y.loc[groups.isin(irregluar_indices)]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
sample_to_fold.update({sample: fold for sample in tmp.index[test]})
 folds = groups.map(group_to_fold)
 is_na = folds.isna()
folds[is_na] = folds[is_na].index.map(sample_to_fold ).values
for i in range(self.n_splits):
yield np.where(folds == i)[0]<load_from_csv>
heads=data.loc[data['parentesco1'] == 1].copy()
train_labels=data.loc[(data['Target'].notnull())&(data['parentesco1']==1),['Target','idhogar']]
dtype = {"cp_type": "category", "cp_dose": "category"}
index_col = "sig_id"
train_features = pd.read_csv(
 ".. /input/lish-moa/train_features.csv", dtype=dtype, index_col=index_col 
)
 test_features = pd.read_csv(
 ".. /input/lish-moa/test_features.csv", dtype=dtype, index_col=index_col 
)
 X = train_features.select_dtypes("number")
 X_test = test_features.select_dtypes("number")
 Y = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv", index_col=index_col)
 groups = pd.read_csv(
 ".. /input/lish-moa/train_drug.csv", index_col=index_col, squeeze=True 
)
 train_size, n_features = X.shape
test_size, _ = X_test.shape
_, n_classes = Y.shape<train_model>
type_counts=train_labels['Target'].value_counts()
n_seeds = 5
n_splits = 5
shuffle = True
params = {"n_layers": 4, "rate": 0.25}
callbacks_params = {"patience_for_es": 20, "save_best_only": True}
fit_params = {"epochs": 1_000}<train_on_grid>
type_counts=pd.DataFrame(type_counts )
%%time
Y_pred = np.zeros(( train_size, n_classes))
Y_pred = pd.DataFrame(Y_pred, columns=Y.columns, index=Y.index)
 for i in range(n_seeds):
set_seed(seed=i)
 cv = MultilabelStratifiedGroupKFold(
 n_splits=n_splits, random_state=i, shuffle=shuffle 
)
 for j,(train, valid)in enumerate(cv.split(X, Y, groups)) :
model_path = f"model_seed_{i}_fold_{j}.h5"
model = build_model(n_features, n_classes, **params)
 callbacks = build_callbacks(model_path, **callbacks_params)
 model.fit(
 X.iloc[train],
Y.iloc[train],
callbacks=callbacks,
validation_data=(X.iloc[valid], Y.iloc[valid]),
**fit_params, 
)
 model.load_weights(model_path)
 Y_pred.iloc[valid] += model.predict(X.iloc[valid])/ n_seeds
Y_pred[train_features["cp_type"] == "ctl_vehicle"] = 0.0<compute_test_metric>
type_counts['level']=type_counts.index
score(Y, Y_pred )<prepare_output>
type_counts['level']=type_counts['level'].replace(poverty_mapping)
%%time
Y_pred = np.zeros(( test_size, n_classes))
Y_pred = pd.DataFrame(Y_pred, columns=Y.columns, index=X_test.index)
 for i in range(n_seeds):
for j in range(n_splits):
model = tf.keras.models.load_model(f"model_seed_{i}_fold_{j}.h5", compile=False)
 Y_pred += model.predict(X_test)/ n_seeds / n_splits
Y_pred[test_features["cp_type"] == "ctl_vehicle"] = 0.0<save_to_csv>
type_counts['Count']=type_counts['Target']
Y_pred.to_csv("submission.csv" )<install_modules>
all_equal=train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() ==1 )
!pip uninstall fastai transformers allennlp kornia tokenizers -y
!pip install.. /input/fastai-2012-wheel/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
!pip install.. /input/fastai-2012-wheel/tokenizers-0.8.1-cp37-cp37m-manylinux1_x86_64.whl
!pip install.. /input/fastai-2012-wheel/kornia-0.4.0-py2.py3-none-any.whl
!pip install.. /input/fastai-2012-wheel/fastcore-1.0.11-py3-none-any.whl
!pip install.. /input/fast-v2-offline/dataclasses-0.6-py3-none-any.whl
!pip install.. /input/fast-v2-offline/torchvision-0.7.0-cp37-cp37m-manylinux1_x86_64.whl
!pip install.. /input/fastai-2012-wheel/fastai-2.0.12-py3-none-any.whl 
<define_variables>
not_equal= all_equal[all_equal!=True]
PATH = '.. /input/lish-moa'
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master' )<set_options>
example=train.loc[train['idhogar']==not_equal.index[0],['idhogar', 'parentesco1', 'Target']]
%pylab inline
print('Fastai: ',fastai.__version__)
 print('Cuda available:', torch.cuda.is_available())
 if torch.cuda.is_available() :
device = torch.cuda.current_device()
torch.cuda.empty_cache()
print('Device:', device)
 else:
device = 'cpu'
plt.rcParams['figure.facecolor'] = 'w'
plt.rcParams['figure.dpi'] = 100
pd.options.display.max_colwidth = 100
pd.options.display.max_columns = 200
def set_random_seed(seed_value, use_cuda=True):
np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 random.seed(seed_value)
 if use_cuda:
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
def rank_gauss(x):
N = x.shape[0]
temp = x.argsort()
rank_x = temp.argsort() / N
rank_x -= rank_x.mean()
rank_x *= 2
efi_x = erfinv(rank_x)
 efi_x -= efi_x.mean()
return efi_x
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
def fe_stats(train, test, features_g, features_c):
print('Adding extra features')
 for df in train, test:
df['e-g_sum'] = df[features_g].sum(axis = 1)
 df['e-g_mean'] = df[features_g].mean(axis = 1)
 df['e-g_std'] = df[features_g].std(axis = 1)
 df['e-g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['e-g_skew'] = df[features_g].skew(axis = 1)
 df['e-c_sum'] = df[features_c].sum(axis = 1)
 df['e-c_mean'] = df[features_c].mean(axis = 1)
 df['e-c_std'] = df[features_c].std(axis = 1)
 df['e-c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['e-c_skew'] = df[features_c].skew(axis = 1)
 df['e-gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['e-gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['e-gc_std'] = df[features_g + features_c].std(axis = 1)
 df['e-gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['e-gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['e-c52_c42'] = df['c-52'] * df['c-42']
df['e-c13_c73'] = df['c-13'] * df['c-73']
df['e-c26_c13'] = df['c-23'] * df['c-13']
df['e-c33_c6'] = df['c-33'] * df['c-6']
df['e-c11_c55'] = df['c-11'] * df['c-55']
df['e-c38_c63'] = df['c-38'] * df['c-63']
df['e-c38_c94'] = df['c-38'] * df['c-94']
df['e-c13_c94'] = df['c-13'] * df['c-94']
df['e-c4_c52'] = df['c-4'] * df['c-52']
df['e-c4_c42'] = df['c-4'] * df['c-42']
df['e-c13_c38'] = df['c-13'] * df['c-38']
df['e-c55_c2'] = df['c-55'] * df['c-2']
df['e-c55_c4'] = df['c-55'] * df['c-4']
df['e-c4_c13'] = df['c-4'] * df['c-13']
df['e-c82_c42'] = df['c-82'] * df['c-42']
df['e-c66_c42'] = df['c-66'] * df['c-42']
df['e-c6_c38'] = df['c-6'] * df['c-38']
df['e-c2_c13'] = df['c-2'] * df['c-13']
df['e-c62_c42'] = df['c-62'] * df['c-42']
df['e-c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'e-{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'e-{feature}_squared'] = df[feature] ** 2
return train, test
def prepare_data(add_pca=False, gaussify=False, variance_threshold=None):
print('Preparing dataset')
 scored = pd.read_csv(f'{PATH}/train_targets_scored.csv' ).set_index('sig_id')
 y_names_scored = scored.columns.to_list()
nonscored = pd.read_csv(f'{PATH}/train_targets_nonscored.csv' ).set_index('sig_id')
 y_names_nonscored = nonscored.columns.to_list()
y_names = y_names_scored + y_names_nonscored
train_feat = pd.read_csv(f'{PATH}/train_features.csv' ).set_index('sig_id')
 cat_names = ['cp_type', 'cp_time', 'cp_dose']
feat_g = train_feat.filter(regex='g-')
 feat_c = train_feat.filter(regex='c-')
 feat_g_names = feat_g.columns.to_list()
feat_c_names = feat_c.columns.to_list()
test_feat = pd.read_csv(f'{PATH}/test_features.csv' ).set_index('sig_id')
 train_feat, test_feat = fe_stats(train_feat, test_feat, feat_g_names, feat_c_names)
 feat_e_names = train_feat.filter(regex='e-' ).columns.to_list()
cont_names = train_feat.filter(regex='g-|c-|e-' ).columns.to_list()
print(f'Cont names(n={len(cont_names)}): {cont_names}')
 feat = train_feat.append(test_feat)
 feat_cont = feat[cont_names]
feat_cat = feat[cat_names]
del feat
if variance_threshold is not None:
print(f'Variance filter {variance_threshold}')
 var_thresh = VarianceThreshold(threshold=variance_threshold)
 transf = var_thresh.fit_transform(feat_cont)
 feat_cont = pd.DataFrame(transf, index=feat_cont.index)\ 
.add_prefix('g-c-e-cont-')
 cont_names = feat_cont.columns.to_list()
feat_cat.loc[:, 'cp_type'] = feat_cat.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 feat_cat.loc[:, 'cp_dose'] = feat_cat.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 feat_cat.loc[:, 'cp_time'] = feat_cat.loc[:, 'cp_time'].map({24: 0, 48: 0.5, 72:1})
 print(len(cont_names))
if gaussify:
print('Gaussify')
 for col in tqdm(cont_names, desc='Gaussify'):
feat_cont.loc[:, col] = rank_gauss(feat_cont[col])
 if add_pca:
print('Add PCA features')
 pca_g = pd.DataFrame(PCA(200 ).fit_transform(feat_g), index=feat_g.index ).add_prefix('g-pca-')
 pca_c = pd.DataFrame(PCA(20 ).fit_transform(feat_c), index=feat_c.index ).add_prefix('c-pca-')
 cont_names.extend(pca_g.columns)
 cont_names.extend(pca_c.columns)
 feat = pd.concat([feat_cat, feat_cont, pca_g, pca_c], axis=1)
 else:
feat = pd.concat([feat_cat, feat_cont], axis=1)
 train = pd.merge(feat.loc[train_feat.index], scored, left_index=True, right_index=True)
 train = pd.merge(train, nonscored, left_index=True, right_index=True)
 test = feat.loc[test_feat.index]
return train, test, y_names, y_names_scored, cont_names, cat_names<split>
households_leader=train.groupby('idhogar')['parentesco1'].sum()
train, test, y_names, y_names_scored, cont_names, cat_names = \
prepare_data(add_pca=True, gaussify=True, variance_threshold=0.85 )<save_to_csv>
households_leader_flase=train.loc[train['idhogar'].isin(households_leader
[households_leader !=1].index),:]
def sigmoid(x):
return 1/(1 + np.exp(-x))
def predict(learn, data):
dl = learn.dls.test_dl(data)
 pred, _ = learn.get_preds(dl=dl)
 pred_scored = pred[:,:].numpy()
return pred_scored
def create_submission(pred):
submission = pd.read_csv(f'{PATH}/sample_submission.csv', index_col=0)
 submission.loc[:, :] = pred
submission.loc[test['cp_type']==1, :] = 0
submission.to_csv('./submission.csv')
 return submission<choose_model_class>
for household in not_equal.index:
true_target= int(train[(train['idhogar']==household)& 
(train['parentesco1']==1.0)]['Target'])
 train.loc[train['idhogar']==household, 'Target']=true_target
class NNet(nn.Module):
def __init__(self, input_dim, layers,
output_dim, sigmoid=False, ps=0,
act_func=nn.ReLU):
super().__init__()
if isinstance(ps, float)or isinstance(ps, int):
ps = [ps]*len(hidden_layer_sizes)
 layers = [input_dim] + layers
last_layer = nn.Linear(layers[-1], output_dim)
 self.layers =\
[nn.Sequential(nn.Dropout(ps_),
nn.utils.weight_norm(nn.Linear(input_, output_)) ,
act_func())
 for input_, output_, ps_ in
zip(layers, layers[1:], ps)] +\
[last_layer]
if sigmoid:
self.layers = self.layers + [nn.Sigmoid() ]
self.layers = nn.Sequential(*self.layers)
 def forward(self, cate_x, cont_x):
x = self.layers(cont_x)
 return x<normalization>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns={0:'total'})
 missing['Percent']=missing['total']/len(data)
 missing.sort_values('Percent',ascending=False ).head(10 )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<split>
heads.groupby('v18q')['v18q1'].apply(lambda x:x.isnull().sum() )
def train_k_fold(train, test, n_folds=5, batch_size=None,
model=None, opt_func=Adam,
fit_pretrain=None,
fit_finetune=None,
use_MLSF=True, y_names=None,
y_names_scored=None,
model_params=None, seed=None):
set_random_seed(seed)
 if use_MLSF:
print('Using Stratified KFold(MLSF)')
 kfold = MultilabelStratifiedKFold(n_folds, shuffle=True, random_state=seed)
 progress = tqdm(enumerate(kfold.split(train, train[y_names])) ,
total=n_folds, desc='MLSF cross-validation')
 else:
print('Using KFold')
 kfold = KFold(n_folds, shuffle=True, random_state=seed)
 progress = tqdm(enumerate(kfold.split(train)) , total=n_folds,
desc='K-fold cross-validation')
 pred = None
for i,(ndx_train, ndx_valid)in progress:
model = NNet(**model_params)
 splits = [ndx_train, ndx_valid]
to = TabularPandas(train, y_names=y_names,
cat_names = None, device=device,
cont_names = cont_names+cat_names,
procs = [Categorify, FillMissing, Normalize],
splits = [list(ndx_train), list(ndx_valid)])
 dls = to.dataloaders(bs=batch_size)
 learn = Learner(dls, model, loss_func=SmoothBCEwLogits(smoothing=0.004),
opt_func=opt_func, cbs=CudaCallback, metrics=BCEWithLogitsLossFlat())
 print('Pretrain')
 fit_pretrain(learn)
 pretrained_model = learn.model
pretrained_model.layers = nn.Sequential(*list(learn.model.layers[:-1]),
nn.Sequential(LinBnDrop(model_params['layers'][-1],
model_params['layers'][-1]),
nn.Linear(model_params['layers'][-1],206)))
 to = TabularPandas(train, y_names=y_names_scored,
cat_names = None, device=device,
cont_names = cont_names+cat_names,
procs = [Categorify, FillMissing, Normalize],
splits = [list(ndx_train), list(ndx_valid)])
 dls = to.dataloaders(bs=batch_size)
 learn = Learner(dls, pretrained_model, loss_func=SmoothBCEwLogits(smoothing=0.004),
opt_func=Adam, cbs=CudaCallback, metrics=BCEWithLogitsLossFlat())
 learn.freeze_to(-1)
 print('Tune')
 fit_finetune(learn)
 valid_value = learn.validate() [0]
print(f'Seed={seed}; Fold={i}: Validation loss:{valid_value:5.4f}')
 _pred = predict(learn, test)
 if pred is None:
pred = _pred
else:
pred += _pred
pred /= n_folds
return pred<split>
data['v18q1']=data['v18q1'].fillna(0 )
train, test, y_names, y_names_scored, cont_names, cat_names = \
prepare_data(add_pca=True, gaussify=True )<train_model>
own_variables=[x for x in data if x.startswith('tipo')]
pred = None
pred_valid = None
n_folds= 5
n_rounds = 2
batch_size = 128
opt_func = Adam
model_params = \
dict(
 input_dim = len(cont_names+cat_names),
output_dim = len(y_names),
layers = [1500, 1250, 1000, 750],
ps = [0.5, 0.35, 0.3, 0.25],
act_func = nn.GELU, 
)
 def fit_pretrain(learn):
cbs = [
EarlyStoppingCallback(monitor='valid_loss', min_delta=1e-5, patience=10),
SaveModelCallback(name='model'),
]
learn.fit_one_cycle(100, 1e-2, cbs=cbs)
 learn.load('model')
 learn.fit_one_cycle(100, slice(1e-3), wd=0.1, cbs=cbs)
 learn.load('model')
 def fit_finetune(learn):
cbs = [
EarlyStoppingCallback(monitor='valid_loss', min_delta=1e-5, patience=10),
SaveModelCallback(name='model'),
]
learn.freeze_to(-1)
 learn.lr_find()
show()
learn.fit_one_cycle(20, slice(1e-4), wd=0.1, cbs=cbs)
 learn.recorder.plot_loss(with_valid=True)
 learn.load('model')
 learn.fit_one_cycle(100, slice(1e-5, 1e-4), wd=0.1, cbs=cbs)
 learn.recorder.plot_loss(with_valid=True)
 learn.load('model')
 learn.freeze_to(-2)
 learn.fit_one_cycle(100, slice(1e-6, 1e-5), wd=0.2, cbs=cbs)
 learn.recorder.plot_loss(with_valid=True)
 learn.load('model')
 show()
pred = None
pred_valid = None
pred_valid_collect = []
losses_valid = []
valid_true = train[y_names_scored].values.flatten()
for i in tqdm(range(n_rounds), desc='Rounds'):
_pred = train_k_fold(train, test, n_folds, batch_size=batch_size,
fit_pretrain=fit_pretrain, fit_finetune=fit_finetune,
opt_func=opt_func, model_params=model_params,
seed=i, y_names=y_names,
y_names_scored=y_names_scored)
 if pred is None:
pred = _pred
else:
pred += _pred
pred /= n_rounds
print(create_submission(sigmoid(pred)) )<define_variables>
owns=pd.DataFrame(data.loc[data['tipovivi3']==1,'Target'] )
DEBUG = False<import_modules>
owns['Target'].value_counts()
sys.path.append('.. /input/iterativestratification')
 !cp -r.. /input/pytorchtabnet/tabnet/*./
os.listdir('.. /input/lish-moa')
<compute_test_metric>
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
def log_loss(y_true, y_pred):
losses = []
for col in range(y.shape[1]):
losses.append(_log_loss(y_true[:,col], y_pred[:,col], eps=1e-15))
return sum(losses)/ y_true.shape[1]<set_options>
data['v2a1-missing'] = data['v2a1'].isnull()
n_comp = 80
Dropout_Model = 0.25
QT_n_quantile_min=10,
QT_n_quantile_max=200
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu' )<load_from_csv>
data['v2a1-missing'].value_counts()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<drop_column>
data['v2a1'] = data['v2a1'].fillna(value=data['tipovivi3'] )
assert(train_features['sig_id'] == train_targets_scored['sig_id'] ).all()
train_features.drop('sig_id', axis=1, inplace=True)
 train_targets_scored.drop('sig_id', axis=1, inplace=True)
 train_targets_nonscored.drop('sig_id', axis=1, inplace=True)
 test_features.drop('sig_id', axis=1, inplace=True )<drop_column>
data.loc[(( data['age']>19)|(data['age']<7)) &(data['rez_esc'].isnull()), 'rez_esc']=0
train_cp = train_features['cp_type'] == 'trt_cp'
test_ctl = test_features['cp_type'] == 'ctl_vehicle'
ctl_df = sample_submission[test_ctl].iloc[:,1:]
ctl_df.values[:,:] = 0 
<define_variables>
data['rez_esc-missing'] = data['rez_esc'].isnull()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
CATEGORICAL = ['cp_type', 'cp_time', 'cp_dose']<categorify>
data['rez_esc']=data['rez_esc'].fillna(0 )
def encode_categorical(data):
data = pd.get_dummies(data, columns=CATEGORICAL)
 return data
train_features.encoded = encode_categorical(train_features)
 test_features.encoded = encode_categorical(test_features )<set_options>
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<categorify>
id_ = ['Id', 'idhogar', 'Target']
print(f"Feature count: {len(train_features.columns)}")
 all_features = pd.concat([train_features.encoded, test_features.encoded])
 def reduce_dimensions(df, n_components, whiten=False):

 pca = PCA(n_components=n_components)
 reduced = pca.fit_transform(df)
 print("
Principal variances
-------------------
" + str(pca.explained_variance_))
return reduced, pca.explained_variance_
temp_, variance = reduce_dimensions(all_features, n_components=130)
 train_features.reduced = temp_[:train_features.shape[0]]
test_features.reduced = temp_[-test_features.shape[0]:]<feature_engineering>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
vt = VarianceThreshold (.9)
 all_features.threshold = vt.fit_transform(all_features.values)
 train_features.threshold = all_features.threshold[:train_features.shape[0]]
test_features.threshold = all_features.threshold[-test_features.shape[0]:]<concatenate>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
train_features.enhanced = np.concatenate(( train_features.threshold, train_features.reduced), axis=1)
 test_features.enhanced = np.concatenate(( test_features.threshold, test_features.reduced), axis=1 )<split>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
def get_folds(X, y, n_splits=7):
mskf = MultilabelStratifiedKFold(n_splits=n_splits)
 return mskf.split(X, y )<data_type_conversions>
heads = data.loc[data['parentesco1'] == 1, :]
heads = heads[id_ + hh_bool + hh_cont + hh_ordered]
heads.shape
class MoADataset:
def __init__(self, features, targets):
self.features = torch.tensor(features, dtype=torch.float, device=DEVICE)
 self.targets = torch.tensor(targets, dtype=torch.float, device=DEVICE)
 def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : self.features[idx],
'y' : self.targets[idx]
}
return dct
class TestDataset:
def __init__(self, features):
self.features = torch.tensor(features, dtype=torch.float, device=DEVICE)
 def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : self.features[idx]
}
return dct<normalization>
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<define_variables>
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True )
EPOCHS = 200
BATCH_SIZE = 1024
LEARNING_RATE =.1e-2
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 100
EARLY_STOP = True
Dropout_model = 0.7
num_features = train_features.reduced.shape[1]
num_targets = len(train_targets_scored.columns)
 hidden_size=200<define_search_model>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
class Chunk(nn.Module):
def __init__(self, in_size, out_size, bn=True, dropout=True):
super().__init__()
self.in_size = in_size
self.out_size = out_size
if bn:
self.bn = nn.BatchNorm1d(in_size)
 else:
self.bn = nn.Identity()
if dropout:
self.dropout = nn.Dropout(Dropout_Model)
 else:
self.dropout = nn.Identity()
self.dense = nn.Linear(in_size, out_size)
 def forward(self, x):
x = self.bn(x)
 x = self.dropout(x)
 x = self.dense(x)
 return x
class Bottleneck(nn.Module):
def __init__(self, size, squeeze_factor=4, **kwargs):
super().__init__()
self.size = size
self.bottleneck_size = size//squeeze_factor
self.chunk1 = Chunk(size, self.bottleneck_size)
 self.chunk2 = Chunk(self.bottleneck_size, size, dropout=False)
 def forward(self, x):
x_skip = x
x = F.leaky_relu(self.chunk1(x))
x = self.chunk2(x)
 return torch.cat([x, x_skip], dim=1)
 class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super().__init__()
self.chunk1 = Chunk(num_features, hidden_size)
 self.chunk2 = Chunk(hidden_size, hidden_size)
 self.chunk3 = Chunk(hidden_size, num_targets)
 def forward(self, x):
x = F.leaky_relu(self.chunk1(x))
x = F.leaky_relu(self.chunk2(x))
x = self.chunk3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<choose_model_class>
heads['walls+roof+floor'] = heads['walls'] + heads['roof'] + heads['floor']
input_dim = train_features.encoded.shape[1]
output_dim = train_targets_scored.values.shape[1]
def get_tabnet(input_dim=input_dim, output_dim=output_dim):
model = TabNet(input_dim, output_dim,
n_d=24, n_a=24,
n_steps=1, n_independent=2, n_shared=0)
 model.to(DEVICE)
 optimizer = optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 loss_fn = nn.BCEWithLogitsLoss()
loss = loss_fn
return model, optimizer, loss<choose_model_class>
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))
def get_scheduler(optimizer, loader, epochs=EPOCHS):
return optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
pct_start=0.1,
div_factor=40,
max_lr=optimizer.defaults['lr'],
epochs=epochs,
steps_per_epoch=len(loader))<choose_model_class>
heads['bonus'] = 1 *(heads['refrig'] +
heads['computer'] + 
(heads['v18q1'] > 0)+
heads['television'])
def get_model(num_features, num_targets, hidden_size):
model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size,
).to(DEVICE)
 optimizer = optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 loss = loss_fn
return model, optimizer, loss
def get_dataloader(*args, batch_size=BATCH_SIZE, shuffle=False):
if len(args)== 1:
dataset = TestDataset(*args)
 elif len(args)== 2:
dataset = MoADataset(*args)
 loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)
 return loader<train_model>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
def train_epoch(model, optimizer, scheduler, loss_fn, dataloader, device=DEVICE, output=None):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'], data['y']
if output is None:
outputs = model(inputs)
 else:
outputs = model(inputs)[output]
loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
@torch.no_grad()
def valid_epoch(model, loss_fn, dataloader, device=DEVICE, output=None):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 if output is None:
outputs = model(inputs)
 else:
outputs = model(inputs)[output]
loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds<train_model>
heads.loc[(heads.v14a == 1)&(heads.sanitario1 == 1)&(heads.abastaguano == 0), "v14a"] = 0
heads.loc[(heads.v14a == 1)&(heads.sanitario1 == 1)&(heads.abastaguano == 0), "sanitario1"] = 0
def train_session(model, train_data, valid_data, epochs=EPOCHS, scheduler=None, optimizer=None,
loss=None, session_name=None, patience=EARLY_STOPPING_STEPS, output=None):
early_step = 0
best_loss = np.inf
session_stats = {'Session': session_name}
if not hasattr(model, 'saved_states'):
model.saved_states = []
for epoch in range(epochs):
train_loss = train_epoch(model, optimizer, scheduler, loss, train_data, output=output)
 valid_loss, valid_preds = valid_epoch(model, loss, valid_data, output=output)
 session_stats.update(
 Epoch=epoch+1,
train_loss=f"{train_loss:.5f}",
valid_loss=f"{valid_loss:.5f}",
lr=f"{scheduler.get_last_lr() [0]:.2}" 
)
 print(", ".join(f"{key}: {value}" for key, value in session_stats.items()))
if valid_loss < best_loss:
early_step -=1
best_loss = valid_loss
torch.save(model.state_dict() , f"{session_name}.pt")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= patience):
break
model.saved_states.append(f"{session_name}.pt")
 def train_kfold(model, optimizer, loss,
x, y,
initialization,
epochs=EPOCHS,
prefix="",
batch_size=BATCH_SIZE,
patience=EARLY_STOPPING_STEPS):
y_pred = np.array(y, dtype=np.float)
 for n, fold in enumerate(get_folds(x, y)) :
session_name = prefix + f" Fold {n+1}" if prefix else f"Fold {n+1}"
train_idx, valid_idx = fold
trainloader = get_dataloader(x[train_idx], y[train_idx], shuffle=True, batch_size=batch_size)
 validloader = get_dataloader(x[valid_idx], y[valid_idx], shuffle=False, batch_size=batch_size)
 model.load_state_dict(initialization)
 scheduler = get_scheduler(optimizer, trainloader, epochs)
 train_session(model, trainloader, validloader,
optimizer=optimizer, loss=loss,
session_name=session_name,
epochs=epochs,
scheduler=scheduler,
patience=patience)
 model.load_state_dict(torch.load(model.saved_states[-1]))
y_pred[valid_idx] = model(validloader.dataset.features ).sigmoid().detach().cpu().numpy()
return y_pred<train_model>
range_ = lambda x: x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()
RUN_PRETRAINING=False
if RUN_PRETRAINING:
x, y = train_features.encoded.values, train_targets_nonscored.values
trainloader = get_dataloader(x, y, shuffle=True, batch_size=2048)
 pretrain_tabnet, optimizer, loss = get_tabnet(input_dim=x.shape[1], output_dim=y.shape[1])
 optimizer.defaults['lr'] =.01
scheduler = get_scheduler(optimizer=optimizer, loader=trainloader, epochs=40)
 train_session(pretrain_tabnet, trainloader, trainloader,
optimizer=optimizer,
loss=loss,
scheduler=scheduler,
session_name="pretrain",
epochs=40,
patience=1)
 pretrain = torch.load("pretrain.pt")
 pretrain.pop("tabnet.final_mapping.weight")
 torch.save(pretrain, "pretrain.pt" )<init_hyperparams>
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()
LOAD_PRETRAIN = False
FREEZE_LAYERS = False
tabnet, optimizer, loss = get_tabnet(input_dim=train_features.encoded.shape[1])
 torch.save(tabnet.state_dict() , "init.pt")
 initialization = torch.load("init.pt")
 if LOAD_PRETRAIN:
pretrain = torch.load("pretrain.pt")
 for key in pretrain.keys() :
initialization[key] = pretrain[key]
if FREEZE_LAYERS:
modules = dict(tabnet.tabnet.named_modules())
 unfreeze_modules = ['final_mapping', 'feat_transformers.1', 'att_transformers.1']
unfreeze_params = [param for name in unfreeze_modules for param in modules[name].parameters() ]
optimizer = optim.Adam(unfreeze_params, lr=2e-02, weight_decay=WEIGHT_DECAY)
 optimizer.defaults['lr'] = 2e-02
x, y = train_features.encoded.values, train_targets_scored.values
y_cv = train_kfold(tabnet, optimizer, loss, x, y, initialization=initialization,
prefix="TabNet", epochs=200, patience=1, batch_size=2048 )<compute_test_metric>
ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )
y_true = train_targets_scored.values
print(f"CV Score: {log_loss(y_true,y_cv)}" )<train_model>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
TRAIN_MLP = True
if TRAIN_MLP:
mlp, optimizer, loss = get_model(num_features=train_features.enhanced.shape[1],
num_targets=train_targets_scored.shape[1], hidden_size=200)
 mlp.saved_states = []
torch.save(mlp.state_dict() , "init.pt")
 optimizer.defaults['lr'] = 0.04
initialization = torch.load("init.pt")
 x, y = train_features.enhanced, train_targets_scored.values
y_cv = train_kfold(mlp, optimizer, loss, x, y, initialization=initialization,
prefix="MLP", epochs=100, patience=1 )<compute_test_metric>
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()
y_true = train_targets_scored.values
print(log_loss(y_true,y_cv))<predict_on_test>
train_set['adult'] = train_set['hogar_adul'] - train_set['hogar_mayor']
train_set['dependency_count'] = train_set['hogar_nin'] + train_set['hogar_mayor']
train_set['dependency'] = train_set['dependency_count'] / train_set['adult']
train_set['child_percent'] = train_set['hogar_nin']/train_set['hogar_total']
train_set['elder_percent'] = train_set['hogar_mayor']/train_set['hogar_total']
train_set['adult_percent'] = train_set['hogar_adul']/train_set['hogar_total']
test_set['adult'] = test_set['hogar_adul'] - test_set['hogar_mayor']
test_set['dependency_count'] = test_set['hogar_nin'] + test_set['hogar_mayor']
test_set['dependency'] = test_set['dependency_count'] / test_set['adult']
test_set['child_percent'] = test_set['hogar_nin']/test_set['hogar_total']
test_set['elder_percent'] = test_set['hogar_mayor']/test_set['hogar_total']
test_set['adult_percent'] = test_set['hogar_adul']/test_set['hogar_total']
train_set['rent_per_adult'] = train_set['v2a1']/train_set['hogar_adul']
train_set['rent_per_person'] = train_set['v2a1']/train_set['hhsize']
test_set['rent_per_adult'] = test_set['v2a1']/test_set['hogar_adul']
test_set['rent_per_person'] = test_set['v2a1']/test_set['hhsize']
train_set['overcrowding_room_and_bedroom'] =(train_set['hacdor'] + train_set['hacapo'])/2
test_set['overcrowding_room_and_bedroom'] =(test_set['hacdor'] + test_set['hacapo'])/2
def predict(x, model):
y_pred_ = []
state_dicts = [torch.load(saved_state)for saved_state in model.saved_states]
for sd in state_dicts:
model.load_state_dict(sd)
 y_pred__ = model(x ).sigmoid().detach().cpu().numpy()
y_pred_.append(y_pred__)
 return sum(y_pred_)/ len(state_dicts )<predict_on_test>
train_set['r4h1_percent_in_male'] = train_set['r4h1'] / train_set['r4h3']
train_set['r4m1_percent_in_female'] = train_set['r4m1'] / train_set['r4m3']
train_set['r4h1_percent_in_total'] = train_set['r4h1'] / train_set['hhsize']
train_set['r4m1_percent_in_total'] = train_set['r4m1'] / train_set['hhsize']
train_set['r4t1_percent_in_total'] = train_set['r4t1'] / train_set['hhsize']
test_set['r4h1_percent_in_male'] = test_set['r4h1'] / test_set['r4h3']
test_set['r4m1_percent_in_female'] = test_set['r4m1'] / test_set['r4m3']
test_set['r4h1_percent_in_total'] = test_set['r4h1'] / test_set['hhsize']
test_set['r4m1_percent_in_total'] = test_set['r4m1'] / test_set['hhsize']
test_set['r4t1_percent_in_total'] = test_set['r4t1'] / test_set['hhsize']
ENSEMBLE = True
target_cols = list(sample_submission.columns[1:])
 test_idx = sample_submission.index
x = {'TabNet': torch.tensor(test_features.encoded.values, dtype=torch.float, device=DEVICE),
'MLP': torch.tensor(test_features.enhanced, dtype=torch.float, device=DEVICE)}
if ENSEMBLE:
models = {'MLP': mlp, 'TabNet': tabnet}
y_preds = []
for name in models:
y_preds.append(predict(x[name], models[name]))
y_pred = sum(y_preds)/ len(models)
 else:
y_pred = predict(x['TabNet'], tabnet)
 predictions_df = pd.DataFrame(y_pred, columns=target_cols)
 sample_submission.update(predictions_df)
 sample_submission.update(ctl_df)
 sample_submission.to_csv('submission.csv', index=False )<define_variables>
train_set['rent_per_room'] = train_set['v2a1']/train_set['rooms']
train_set['bedroom_per_room'] = train_set['bedrooms']/train_set['rooms']
train_set['elder_per_room'] = train_set['hogar_mayor']/train_set['rooms']
train_set['adults_per_room'] = train_set['adult']/train_set['rooms']
train_set['child_per_room'] = train_set['hogar_nin']/train_set['rooms']
train_set['male_per_room'] = train_set['r4h3']/train_set['rooms']
train_set['female_per_room'] = train_set['r4m3']/train_set['rooms']
train_set['room_per_person_household'] = train_set['hhsize']/train_set['rooms']
test_set['rent_per_room'] = test_set['v2a1']/test_set['rooms']
test_set['bedroom_per_room'] = test_set['bedrooms']/test_set['rooms']
test_set['elder_per_room'] = test_set['hogar_mayor']/test_set['rooms']
test_set['adults_per_room'] = test_set['adult']/test_set['rooms']
test_set['child_per_room'] = test_set['hogar_nin']/test_set['rooms']
test_set['male_per_room'] = test_set['r4h3']/test_set['rooms']
test_set['female_per_room'] = test_set['r4m3']/test_set['rooms']
test_set['room_per_person_household'] = test_set['hhsize']/test_set['rooms']
train_set['rent_per_bedroom'] = train_set['v2a1']/train_set['bedrooms']
train_set['edler_per_bedroom'] = train_set['hogar_mayor']/train_set['bedrooms']
train_set['adults_per_bedroom'] = train_set['adult']/train_set['bedrooms']
train_set['child_per_bedroom'] = train_set['hogar_nin']/train_set['bedrooms']
train_set['male_per_bedroom'] = train_set['r4h3']/train_set['bedrooms']
train_set['female_per_bedroom'] = train_set['r4m3']/train_set['bedrooms']
train_set['bedrooms_per_person_household'] = train_set['hhsize']/train_set['bedrooms']
test_set['rent_per_bedroom'] = test_set['v2a1']/test_set['bedrooms']
test_set['edler_per_bedroom'] = test_set['hogar_mayor']/test_set['bedrooms']
test_set['adults_per_bedroom'] = test_set['adult']/test_set['bedrooms']
test_set['child_per_bedroom'] = test_set['hogar_nin']/test_set['bedrooms']
test_set['male_per_bedroom'] = test_set['r4h3']/test_set['bedrooms']
test_set['female_per_bedroom'] = test_set['r4m3']/test_set['bedrooms']
test_set['bedrooms_per_person_household'] = test_set['hhsize']/test_set['bedrooms']
train_set['tablet_per_person_household'] = train_set['v18q1']/train_set['hhsize']
train_set['phone_per_person_household'] = train_set['qmobilephone']/train_set['hhsize']
test_set['tablet_per_person_household'] = test_set['v18q1']/test_set['hhsize']
test_set['phone_per_person_household'] = test_set['qmobilephone']/test_set['hhsize']
train_set['age_12_19'] = train_set['hogar_nin'] - train_set['r4t1']
test_set['age_12_19'] = test_set['hogar_nin'] - test_set['r4t1']
DO_SUBMISSION = True
DO_TRAIN_FOR_ENSEMBLE = False
DO_VIRTUAL_SUBMISSION = False
assert(sum([DO_SUBMISSION, DO_TRAIN_FOR_ENSEMBLE, DO_VIRTUAL_SUBMISSION])== 1), "select `ONE` mode"<install_modules>
train_set['num_over_18'] = 0
train_set['num_over_18'] = train_set[train.age >= 18].groupby('idhogar' ).transform("count")
 train_set['num_over_18'] = train_set.groupby("idhogar")["num_over_18"].transform("max")
 train_set['num_over_18'] = train_set['num_over_18'].fillna(0)
 test_set['num_over_18'] = 0
test_set['num_over_18'] = test_set[test.age >= 18].groupby('idhogar' ).transform("count")
 test_set['num_over_18'] = test_set.groupby("idhogar")["num_over_18"].transform("max")
 test_set['num_over_18'] = test_set['num_over_18'].fillna(0 )
%%time
!pip install --no-index --find-links.. /input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<install_modules>
train_set=train_set.drop(columns='idhogar')
 test_set=test_set.drop(columns='idhogar' )
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/<set_options>
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v14a']
train_set = train_set.drop(needless_cols, axis=1)
 test_set = test_set.drop(needless_cols, axis=1 )
warnings.resetwarnings()
warnings.simplefilter('ignore', FutureWarning)
 warnings.simplefilter('ignore', DeprecationWarning)
 os.environ["CUDA_VISIBLE_DEVICES"]="0"<define_variables>
train_set= train_set.fillna(0)
 test_set= test_set.fillna(0 )
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat<define_variables>
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'] )
MODEL_NAMES = ["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]<load_pretrained>
train_set = pd.DataFrame(train_set, columns = features)
 corr_matrix = train_set.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 1)]
to_drop
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<load_pretrained>
test_set = pd.DataFrame(test_set, columns = features)
 train_set, test_set = train_set.align(test_set, axis = 1, join = 'inner')
 features = list(train_set.columns )
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
estimator = RandomForestClassifier(random_state = 10, n_estimators = 100, n_jobs = -1)
 selector = RFECV(estimator, step = 1, cv = 3, scoring= scorer, n_jobs = -1 )
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
selector.fit(train_set, train_labels )
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<define_variables>
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']<load_from_csv>
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected, columns = selected_features)
 test_selected = pd.DataFrame(test_selected, columns = selected_features )
def del_control(df: pd.DataFrame())-> pd.DataFrame() :
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 df['cp_type'] = train['cp_type']
df = df.loc[df['cp_type']=='trt_cp'].reset_index(drop=True)
 df = df.drop('cp_type', axis=1)
 return df
def drop_sig_id(df: pd.DataFrame())-> pd.DataFrame() :
return df.drop('sig_id', axis=1)
 def fix_oof(df: pd.DataFrame())-> pd.DataFrame() :
if df.shape[0] != 21948:
df = del_control(df)
 if 'sig_id' in df.columns.tolist() :
df = drop_sig_id(df)
 return df<load_from_csv>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
oof_list = []
oof_list.append(fix_oof(
 pd.read_pickle('.. /input/nn-use-train-public/oof.pkl')))
 oof_list.append(fix_oof(
 pd.DataFrame(
 np.load('.. /input/tabnet-train-public-add-n-shared-1/oof_tabnet.npy'),
columns=TARGET_COL)) 
)
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/oof_prediction_takapy_resnet.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-seed-cv/oof_prediction.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-drug-seed-cv/oof_prediction.csv")) )<load_from_csv>
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
y_true = fix_oof(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
for i, name in enumerate(MODEL_NAMES):
print(f"[{name}]")
 print("shape:", oof_list[i].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[i].values.ravel()))<categorify>
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None,random_state_int=101,early_stopping=300):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = 4, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True, random_state= random_state_int)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds =early_stopping,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
callbacks=[lgb.reset_parameter(learning_rate=learning_rate_power_0997)])
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
class MoAStackingDatasetForCNN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][..., self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDatasetForGCN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]<define_search_model>
hyp1 = {
'colsample_bytree': 0.88,
'learning_rate': 0.08,
'min_child_samples': 90, 'num_leaves': 34, 'subsample': 0.94, 'reg_lambda': 0.5,
'class_weight': 'balanced'}
hyp2 = {
'colsample_bytree': 0.88,
'learning_rate': 0.08,
'min_child_samples': 90, 'num_leaves': 14, 'subsample': 0.94, 'reg_lambda': 0.5,
'class_weight': 'balanced'}
hyp3 = {
'colsample_bytree': 0.78,
'learning_rate': 0.08,
'min_child_samples': 45, 'num_leaves': 14, 'subsample': 0.64, 'reg_lambda': 0.1,
'class_weight': 'balanced'}
hyp4 = {
'colsample_bytree': 0.72,
'learning_rate': 0.08,
'min_child_samples': 30, 'num_leaves': 18, 'subsample': 0.64, 'reg_lambda': 0.1,
'class_weight': 'balanced'}
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )<prepare_x_and_y>
%%capture --no-display
submission1, gbm_fi_selected, valid_scores_selected = model_gbm(train_set, train_labels, test_set, test_ids,hyp = hyp1,random_state_int=103,early_stopping=300)
 submission2, gbm_fi_selected, valid_scores_selected = model_gbm(train_set, train_labels, test_set, test_ids,hyp = hyp1,random_state_int=103,early_stopping=300)
 submission3, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids)
def vector_wise_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 X = torch.transpose(X, 0, 1)
 W = torch.transpose(W, 1, 2)
 H = torch.matmul(X, W)
 H = torch.transpose(H, 0, 1)
 return H
def vector_wise_shared_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 H = nn.functional.linear(X, W)
 return H<normalization>
def model_gbm_2(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None,random_state_int=101,early_stopping=300):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = 4, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True, random_state= random_state_int)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds =early_stopping,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
def _calculate_fan_in_and_fan_out_for_vwl(tensor)-> tp.Tuple[int]:

 dimensions = tensor.dim()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
 fan_in = tensor.size(-1)
 fan_out = tensor.size(-2)
 return fan_in, fan_out
def _calculate_correct_fan_for_vwl(tensor, mode)-> int:

 mode = mode.lower()
valid_modes = ['fan_in', 'fan_out']
if mode not in valid_modes:
raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
fan_in, fan_out = _calculate_fan_in_and_fan_out_for_vwl(tensor)
 return fan_in if mode == 'fan_in' else fan_out
def kaiming_uniform_for_vwl(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'):

 fan = _calculate_correct_fan_for_vwl(tensor, mode)
 gain = nn.init.calculate_gain(nonlinearity, a)
 std = gain / np.sqrt(fan)
 bound = np.sqrt(3.0)* std
with torch.no_grad() :
return tensor.uniform_(-bound, bound )<prepare_x_and_y>
%%capture --no-display
submission4, gbm_fi_selected, valid_scores_selected = model_gbm_2(train_set, train_labels, test_set, test_ids,hyp = hyp3,random_state_int=103,early_stopping=300 )
class VectorWiseLinear(nn.Module):

 def __init__(
 self,
in_features: int, out_features: int, n_vectors: int,
bias: bool=True, weight_shared: bool=True 
)-> None:

 super(VectorWiseLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.n_vectors = n_vectors
self.weight_shared = weight_shared
if self.weight_shared:
self.weight = nn.Parameter(
 torch.Tensor(self.out_features, self.in_features))
self.matmul_func = vector_wise_shared_matmul
else:
self.weight = nn.Parameter(
 torch.Tensor(self.n_vectors, self.out_features, self.in_features))
self.matmul_func = vector_wise_matmul
if bias:
self.bias = nn.Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 self.reset_parameters()
def reset_parameters(self)-> None:

 kaiming_uniform_for_vwl(self.weight, a=np.sqrt(5))
if self.bias is not None:
fan_in, _ = _calculate_fan_in_and_fan_out_for_vwl(self.weight)
 bound = 1 / np.sqrt(fan_in)
 nn.init.uniform_(self.bias, -bound, bound)
 def forward(self, X: torch.Tensor)-> torch.Tensor:

 H = self.matmul_func(X, self.weight)
 if self.bias is not None:
H = H + self.bias
return H<import_modules>
suball=submission1.merge(submission2, on='Id')
 suball=suball.merge(submission4, on='Id')
 suball=suball.merge(submission3, on='Id')
 suball_2=suball.merge(submission4, on='Id' )
class GraphConv(nn.Module):

 def __init__(
 self,
in_channels: int, out_channels: int, n_nodes: int, shrare_msg: bool=True,
model_self: bool=True, share_model_self: bool=True,
bias: bool=True, share_bias: bool=True 
)-> None:

 self.in_channels = in_channels
self.out_channels = out_channels
self.n_nodes = n_nodes
self.model_self = model_self
super(GraphConv, self ).__init__()
self.msg = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, shrare_msg)
 if model_self:
self.model_self = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, share_model_self)
 if bias:
if share_bias:
self.bias = nn.Parameter(torch.Tensor(out_channels))
else:
self.bias = nn.Parameter(torch.Tensor(n_nodes, out_channels))
bound = 1 / np.sqrt(out_channels)
 nn.init.uniform_(self.bias, -bound, bound)
 else:
self.register_parameter('bias', None)
 def forward(self, X: torch.Tensor, A: torch.Tensor, W: torch.Tensor=None)-> torch.Tensor:

 if W is not None:
A = A * W
M = X
M = self.msg(M)
 M = torch.matmul(A, M)
 H = M
if self.model_self:
H = H + self.model_self(X)
 if self.bias is not None:
H = H + self.bias
return H<choose_model_class>
suball_2['index']=suball_2.index
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNStacking1d(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking1d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(
 in_ch, out_ch, kernel_size=3, stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class CNNStacking2d(nn.Module):

 def __init__(
 self, n_models: int, n_classes: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking2d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv2d(
 in_ch, out_ch, kernel_size=(1, 3), stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class GCNStacking(nn.Module):

 def __init__(
 self, n_classes: int,
n_channels_list: tp.List[int],
add_self_loop: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(GCNStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = GraphConv(
 in_ch, out_ch, n_classes,
shrare_msg=False, share_model_self=False, share_bias=False)
 setattr(self, "conv{}".format(i + 1), layer)
 self.relu = nn.ReLU(inplace=True)
 if add_self_loop:
adj_mat = torch.ones(n_classes, n_classes)/ n_classes
else:
adj_mat =(1 - torch.eye(n_classes, n_classes)) /(n_classes - 1)
 self.register_buffer("A", adj_mat.float())
 kwargs_head["n_features_list"][0] = n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, X: torch.FloatTensor)-> torch.Tensor:

 bs, n_classes = X.shape[:2]
H = X
for i in range(self.n_conv_layers):
H = getattr(self, "conv{}".format(i + 1))(H, self.A[None,...])
 H = self.relu(H)
 h = torch.reshape(H,(bs, -1))
h = self.head(h)
 return h<init_hyperparams>
modle_13= pd.DataFrame(suball_2.mode(axis=1)[0] )
class EvalFuncManager(nn.Module):

 def __init__(
 self, iters_per_epoch: int,
evalfunc_dict: tp.Dict[str, nn.Module], prefix: str = "val" 
)-> None:

 super(EvalFuncManager, self ).__init__()
self.tmp_iter = 0
self.iters_per_epoch = iters_per_epoch
self.prefix = prefix
self.metric_names = []
for k, v in evalfunc_dict.items() :
setattr(self, k, v)
 self.metric_names.append(k)
 self.reset()
def reset(self)-> None:

 self.tmp_iter = 0
for name in self.metric_names:
getattr(self, name ).reset()
def __call__(self, y: torch.Tensor, t: torch.Tensor)-> None:

 for name in self.metric_names:
getattr(self, name ).update(y, t)
 self.tmp_iter += 1
if self.tmp_iter == self.iters_per_epoch:
ppe.reporting.report({
"{}/{}".format(self.prefix, name): getattr(self, name ).compute()
for name in self.metric_names})
 self.reset()
class MeanLoss(nn.Module):
def __init__(self):
super(MeanLoss, self ).__init__()
self.loss_sum = 0
self.n_examples = 0
def forward(self, y: torch.Tensor, t: torch.Tensor):

 return self.loss_func(y, t)
 def reset(self):

 self.loss_sum = 0
self.n_examples = 0
def update(self, y: torch.Tensor, t: torch.Tensor):

 self.loss_sum += self(y, t ).item() * y.shape[0]
self.n_examples += y.shape[0]
def compute(self):

 return self.loss_sum / self.n_examples
class MyLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLogLoss, self ).__init__()
self.loss_func = nn.BCEWithLogitsLoss(**params)
 class LSBCEWithLogitsLoss(nn.Module):

 def __init__(self, k: int, alpha: float=0.01):

 super(LSBCEWithLogitsLoss, self ).__init__()
self.k = k
self.alpha = alpha
self.loss_func = nn.BCEWithLogitsLoss()
def forward(self, y, t):

 t_s = t *(1 - self.alpha)+ self.alpha / self.k
loss = self.loss_func(y, t_s)
 return loss
class MyLSLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLSLogLoss, self ).__init__()
self.loss_func = LSBCEWithLogitsLoss(**params )<train_model>
modle_13['index']=modle_13.index
def run_train_loop(
 manager, args, model, device,
train_loader, optimizer, scheduler, loss_func 
):

 while not manager.stop_trigger:
model.train()
for batch in train_loader:
x, t = batch
with manager.run_iteration() :
optimizer.zero_grad()
y = model(x.to(device))
loss = loss_func(y, t.to(device))
ppe.reporting.report({'train/loss': loss.item() })
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
def run_eval(args, model, device, batch, eval_manager):

 model.eval()
x, t = batch
y = model(x.to(device))
eval_manager(y, t.to(device))<choose_model_class>
final = suball_2.merge(modle_13, on ='index' )
def get_optimizer(settings, model):
optimizer = getattr(torch.optim, settings["optimizer"]["name"] )(
 model.parameters() , **settings["optimizer"]["params"])
 return optimizer
def get_scheduler(settings, train_loader, optimizer):
if settings["scheduler"]["name"] is None:
scheduler = None
else:
if settings["scheduler"]["name"] == "OneCycleLR":
settings["scheduler"]["params"]["epochs"] = settings["globals"]["max_epoch"]
settings["scheduler"]["params"]["steps_per_epoch"] = len(train_loader)
 scheduler = getattr(torch.optim.lr_scheduler, settings["scheduler"]["name"] )(
 optimizer, **settings["scheduler"]["params"])
 return scheduler
def get_loss_function(settings):
if hasattr(nn, settings["loss"]["name"]):
loss_func = getattr(nn, settings["loss"]["name"] )(**settings["loss"]["params"])
 else:
loss_func = eval(settings["loss"]["name"] )(**settings["loss"]["params"])
 return loss_func
def get_manager(
 settings, model, device, train_loader, val_loader, optimizer, eval_manager, output_path 
):
trigger = ppe.training.triggers.EarlyStoppingTrigger(
 check_trigger=(1, 'epoch'), monitor='val/metric', mode="min",
patience=settings["globals"]["patience"], verbose=True,
max_trigger=(settings["globals"]["max_epoch"], 'epoch'))
manager = ppe.training.ExtensionsManager(
 model, optimizer, settings["globals"]["max_epoch"],
iters_per_epoch=len(train_loader), stop_trigger=trigger, out_dir=output_path)
 log_extentions = [
ppe_extensions.observe_lr(optimizer=optimizer),
ppe_extensions.LogReport() ,
ppe_extensions.PlotReport(["train/loss", "val/loss"], 'epoch', filename='loss.png'),
ppe_extensions.PlotReport(["lr"], 'epoch', filename='lr.png'),
ppe_extensions.PrintReport([
"epoch", "iteration", "lr", "train/loss", "val/loss", "val/metric", "elapsed_time"])
 ]
for ext in log_extentions:
manager.extend(ext)
 manager.extend(
 ppe_extensions.Evaluator(
 val_loader, model,
eval_func=lambda *batch: run_eval(settings, model, device, batch, eval_manager)) ,
trigger=(1, "epoch"))
manager.extend(
 ppe_extensions.snapshot(target=model, filename="snapshot_epoch_{.epoch}.pth"),
trigger=ppe.training.triggers.MinValueTrigger(key="val/metric", trigger=(1, 'epoch')))
 return manager<set_options>
final_res=final[['Id',0]]
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True<train_model>
final_res.columns=['Id','Target']
<groupby><EOS> 
final_res.to_csv('4_model_vote.csv',index=False )