Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<define_variables>
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import torch.nn as nn
import torch.optim as optim
import torchvision
from torch.autograd import Variable
import torch
g_features = [col for col in train_features.columns if col.startswith('g-')]
c_features = [col for col in train_features.columns if col.startswith('c-')]
g_c_features = g_features + c_features<feature_engineering>
def F1score(actuals, preds):

 return f1_score(actuals, preds, average = 'macro' )
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
<create_dataframe>
PATH = ".. /input/"
train = pd.read_csv(f'{PATH}train.csv')
 test = pd.read_csv(f'{PATH}test.csv')
 train.info()
trans_train_features = transformer.fit_transform(train_features[g_c_features])
 trans_test_features = transformer.transform(test_features[g_c_features])
 trans_train_df = pd.DataFrame(trans_train_features, columns = g_c_features)
 trans_test_df = pd.DataFrame(trans_test_features, columns = g_c_features)
 train_features = pd.concat([train_features.drop(columns=g_c_features), trans_train_df], axis=1)
 test_features = pd.concat([test_features.drop(columns=g_c_features), trans_test_df], axis=1 )<categorify>
obj_cols = train.columns[train.dtypes == "object"]; obj_cols
def transfrom_all_data(transformer, train, test, feature_list):
data = pd.concat([train[feature_list], test[feature_list]], axis=0 ).reset_index(drop=True)
 n = train.shape[0]
data_trans = transformer.fit_transform(data)
 train_trans = data_trans[:n, :]
test_trans = data_trans[n:, :]
return train_trans, test_trans<normalization>
lb1 = LabelEncoder()
lb1.fit(list(train['Id'].values))
lb2 = LabelEncoder()
lb2.fit(list(train['idhogar'].values))
train['Id'] = lb1.transform(list(train['Id'].values))
train['idhogar'] = lb2.transform(list(train['idhogar'].values))
lb3 = LabelEncoder()
lb3.fit(list(test['Id'].values))
lb4 = LabelEncoder()
lb4.fit(list(test['idhogar'].values))
test['Id'] = lb3.transform(list(test['Id'].values))
test['idhogar'] = lb4.transform(list(test['idhogar'].values))
def make_pca_features(n_comp, train, test, feature_list, name, normalize=False, scaler=None):
pca = PCA(n_comp)
 train_pca, test_pca = transfrom_all_data(pca, train, test, feature_list)
 if normalize and scaler is not None:
train_pca = scaler.fit_transform(train_pca)
 test_pca = scaler.transform(test_pca)
 for i in range(n_comp):
train['{0}_{1}'.format(name, i)] = train_pca[:, i]
test['{0}_{1}'.format(name, i)] = test_pca[:, i]
return train, test<categorify>
train['dependency'].replace('yes', '1', inplace=True)
 train['dependency'].replace('no', '0', inplace=True)
 train['dependency'].astype(np.float64);
def preprocess(data):
data['cp_time'] = data['cp_time'].map({24:0, 48:1, 72:2})
 data['cp_dose'] = data['cp_dose'].map({'D1':0, 'D2':1})
 return data<concatenate>
test['dependency'].replace('yes', '1', inplace=True)
 test['dependency'].replace('no', '0', inplace=True)
 test['dependency'].astype(np.float64);
train_features, test_features = make_pca_features(params['n_comp_g'], train_features, test_features, g_features, 'g_pca')
<concatenate>
train['edjefe'].replace('yes', '1', inplace=True)
 train['edjefe'].replace('no', '0', inplace=True)
 train['edjefe'].astype(np.float64);
train_features, test_features = make_pca_features(params['n_comp_c'], train_features, test_features, c_features, 'c_pca' )<categorify>
test['edjefe'].replace('yes', '1', inplace=True)
 test['edjefe'].replace('no', '0', inplace=True)
 test['edjefe'].astype(np.float64);
train_thresh, test_thresh = transfrom_all_data(var_thresh, train_features, test_features, to_thresh )<concatenate>
train['edjefa'].replace('yes', '1', inplace=True)
 train['edjefa'].replace('no', '0', inplace=True)
 train['edjefa'].astype(np.float64);
train_features = pd.concat([train_features[cat_features], pd.DataFrame(train_thresh)], axis=1)
 test_features = pd.concat([test_features[cat_features], pd.DataFrame(test_thresh)], axis=1 )<prepare_x_and_y>
test['edjefa'].replace('yes', '1', inplace=True)
 test['edjefa'].replace('no', '0', inplace=True)
 test['edjefa'].astype(np.float64);
train_mask = train_features['cp_type'] != 'ctl_vehicle'
train_sig_ids = train_features.loc[train_mask]['sig_id']
train = train_features.loc[train_mask].reset_index(drop=True)
 test_mask = test_features['cp_type'] != 'ctl_vehicle'
test_sig_ids = test_features.loc[test_mask]['sig_id']
test = test_features.loc[test_mask].reset_index(drop=True)
 train_target_sigids = train_targets[['sig_id']]
y_true = train_targets.copy()
train_targets = train_targets[train_targets['sig_id'].isin(train_sig_ids)].reset_index(drop=True)
 train_targets.drop(columns=['sig_id'], inplace=True)
 train_targets.reset_index(drop=True, inplace=True )<init_hyperparams>
null_counts = train.isnull().sum()
null_counts[null_counts>0]
params['in_size'] = train.shape[1] - 2
params['out_size'] = train_targets.shape[1]<choose_model_class>
test_null_counts = test.isnull().sum()
test_null_counts[test_null_counts>0]
mskf = MultilabelStratifiedKFold(n_splits=params['n_folds'] )<split>
cols = ['Id', 'parentesco1', 'v2a1', 'v18q', 'hacapo', 'rooms', 'r4t3', 'hhsize', 'escolari', 'epared2',
'epared3', 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5', 'Target']
folds = train.copy()
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train_targets)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )<data_type_conversions>
v2a1_null_heads.query('tipovivi1 != 1 & tipovivi2 != 1 & tipovivi3 != 1 & tipovivi4 != 1 & tipovivi5 != 1')
class TabularDataset:
def __init__(self, X, y):
self.X = X
self.y = y
def __len__(self):
return(self.X.shape[0])
 def __getitem__(self, i):
X_i = torch.tensor(self.X[i, :], dtype=torch.float)
 y_i = torch.tensor(self.y[i, :], dtype=torch.float)
 return X_i, y_i
class TabularDatasetTest:
def __init__(self, X):
self.X = X
def __len__(self):
return(self.X.shape[0])
 def __getitem__(self, i):
X_i = torch.tensor(self.X[i, :], dtype=torch.float)
 return X_i<train_model>
train.loc[train['v2a1'].isnull() & train['tipovivi1'] == 1, 'v2a1'] = 0
def train_func(model, optimizer, scheduler, loss_func, dataloader, device):
train_loss = 0
model.train()
for inputs, labels in dataloader:
optimizer.zero_grad()
inputs, labels = inputs.to(device), labels.to(device)
 outputs = model(inputs)
 loss = loss_func(outputs, labels)
 loss.backward()
optimizer.step()
scheduler.step()
train_loss += loss.item()
train_loss /= len(dataloader)
 return train_loss<compute_train_metric>
test.loc[test['v2a1'].isnull() & test['tipovivi1'] == 1, 'v2a1'] = 0
def valid_func(model, loss_func, dataloader, device):
model.eval()
valid_loss = 0
valid_preds = []
for inputs, labels in dataloader:
inputs, labels = inputs.to(device), labels.to(device)
 outputs = model(inputs)
 loss = loss_func(outputs, labels)
 valid_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 valid_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return valid_loss, valid_preds<predict_on_test>
train.loc[train['v2a1'].isnull() &(train['Target']== 1), 'v2a1'] = a
train.loc[train['v2a1'].isnull() &(train['Target']== 2), 'v2a1'] = b
train.loc[train['v2a1'].isnull() &(train['Target']== 3), 'v2a1'] = c
train.loc[train['v2a1'].isnull() &(train['Target']== 4), 'v2a1'] = d
train.loc[train['v2a1'].isnull() ]
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data.to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
test.loc[test['v2a1'].isnull() , 'v2a1'] =(a+b+c+d)/4
test.loc[test['v2a1'].isnull() ]
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<normalization>
h_ids = v18q1_null['idhogar'].unique() ; h_ids.shape
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.25)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x 
<prepare_x_and_y>
for idn in h_ids:
train.loc[(train['idhogar'] == idn), 'v18q1'] = train.query(f'idhogar == {idn}')['v18q'].sum()
def run_training(fold, seed):
seed_everything(seed)
 train = preprocess(folds.drop(columns = ['sig_id', 'cp_type']))
train_mask = train['kfold'] != fold
valid_idc = train.loc[~train_mask].index
X_train = train.loc[train_mask].reset_index(drop=True)
 y_train = train_targets.loc[train_mask].reset_index(drop=True)
 X_val = train.loc[~train_mask].reset_index(drop=True)
 y_val = train_targets.loc[~train_mask].reset_index(drop=True)
 X_train.drop(columns=['kfold'], inplace=True)
 X_val.drop(columns=['kfold'], inplace=True)
 test_ = preprocess(test.drop(columns = ['sig_id', 'cp_type']))
train_ds = TabularDataset(X_train.values, y_train.values)
 valid_ds = TabularDataset(X_val.values, y_val.values)
 test_ds = TabularDatasetTest(test_.values)
 train_dl = DataLoader(train_ds, batch_size=params['batch_size'], shuffle=True)
 valid_dl = DataLoader(valid_ds, batch_size=params['batch_size'], shuffle=False)
 test_dl = DataLoader(test_ds, batch_size=params['batch_size'], shuffle=False)
 model = Model(num_features=params['in_size'], num_targets=params['out_size'],
hidden_size=params['hidden_size'])
 model.to(params['device'])
 optimizer = torch.optim.Adam(model.parameters() , lr=params['lr'], weight_decay=params['weight_decay'])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=params['epochs'], steps_per_epoch=len(train_dl))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 early_stopping_steps = params['early_stopping_steps']
early_step = 0
oof = np.zeros(( train.shape[0], params['out_size']))
best_loss = np.inf
for epoch in range(params['epochs']):
train_loss = train_func(model, optimizer,scheduler, loss_tr, train_dl, params['device'])
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_func(model, loss_fn, valid_dl, params['device'])
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[valid_idc] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(params['early_stop'] == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
model = Model(num_features=params['in_size'], num_targets=params['out_size'],
hidden_size=params['hidden_size'])
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(params['device'])
 predictions = np.zeros(( test.shape[0], params['out_size']))
predictions = inference_fn(model, test_dl, params['device'])
 return oof, predictions<categorify>
for idn in h_ids:
test.loc[(test['idhogar'] == idn), 'v18q1'] = test.query(f'idhogar == {idn}')['v18q'].sum()
def run_k_fold(n_folds, seed):
oof = np.zeros(( train.shape[0], params['out_size']))
predictions = np.zeros(( test.shape[0], params['out_size']))
for fold in range(n_folds):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / n_folds
oof += oof_
return oof, predictions<set_options>
cols = ['Id', 'idhogar', 'escolari', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4',
'estadocivil5', 'estadocivil6', 'estadocivil7', 'instlevel1', 'age', 'Target']
cols2 = cols[:-1]
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<define_variables>
train.loc[(train['rez_esc'].isnull())&(train['instlevel1'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['instlevel1'] == 1), 'rez_esc'] = 0
seeds = [0, 1, 2, 3, 4, 5]
oof = np.zeros(( train.shape[0], params['out_size']))
predictions = np.zeros(( test.shape[0], params['out_size']))
for seed in seeds:
oof_, predictions_ = run_k_fold(params['n_folds'], seed)
 oof += oof_ / len(seeds)
 predictions += predictions_ / len(seeds )<concatenate>
a = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil2'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
b = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil3'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
c = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil4'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
d = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil5'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
e = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil6'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
f = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil7'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
a, b, c, d, e, f
train.loc[(train['rez_esc'].isnull())&(train['estadocivil2'] == 1), 'rez_esc'] = 3
train.loc[(train['rez_esc'].isnull())&(train['estadocivil3'] == 1), 'rez_esc'] = 0
train.loc[(train['rez_esc'].isnull())&(train['estadocivil4'] == 1), 'rez_esc'] = 0
train.loc[(train['rez_esc'].isnull())&(train['estadocivil5'] == 1), 'rez_esc'] = 2
train.loc[(train['rez_esc'].isnull())&(train['estadocivil6'] == 1), 'rez_esc'] = 0
train.loc[(train['rez_esc'].isnull())&(train['estadocivil7'] == 1), 'rez_esc'] = 1
test.loc[(test['rez_esc'].isnull())&(test['estadocivil2'] == 1), 'rez_esc'] = 3
test.loc[(test['rez_esc'].isnull())&(test['estadocivil3'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['estadocivil4'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['estadocivil5'] == 1), 'rez_esc'] = 2
test.loc[(test['rez_esc'].isnull())&(test['estadocivil6'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['estadocivil7'] == 1), 'rez_esc'] = 1
valid_results = pd.concat([train_target_sigids[train_target_sigids['sig_id'].isin(train_sig_ids)].reset_index(drop=True), pd.DataFrame(oof)], axis=1 )<concatenate>
train['rez_esc'].isnull().sum() , test['rez_esc'].isnull().sum()
test_results = pd.concat([test[['sig_id']], pd.DataFrame(predictions, columns = sample_submission.columns[1:])], axis=1 )<merge>
cols = ['Id', 'idhogar', 'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']
valid_full = train_target_sigids.merge(valid_results, on='sig_id', how='left' ).fillna(0)
<compute_test_metric>
print(train.loc[(train['idhogar'] == 326), 'meaneduc'].values)
 print(train.loc[(train['idhogar'] == 1959), 'meaneduc'].values)
 print(train.loc[(train['idhogar'] == 2908), 'meaneduc'].values )
y_true = y_true.drop(columns=['sig_id'] ).values
y_pred = valid_full.drop(columns=['sig_id'] ).values
score = 0
for i in range(y_true.shape[1]):
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / y_true.shape[1]
print("CV log_loss: ", score )<save_to_csv>
def meaneduc_correction(null_view, df, hids):

 for idn in hids:
a = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel1'] == 1)].shape[0]
b = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel2'] == 1)].shape[0]
c = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel3'] == 1)].shape[0]
d = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel4'] == 1)].shape[0]
e = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel5'] == 1)].shape[0]
f = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel6'] == 1)].shape[0]
g = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel7'] == 1)].shape[0]
h = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel8'] == 1)].shape[0]
i = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel9'] == 1)].shape[0]
mean_educ =(a*0 + b*4 + c*8 + d*2 + e*4 + f + g*2 + h*4 + i)/(a+b+c+d+e+f+g+h+i)
 df.loc[(df['meaneduc'].isnull())&(df['idhogar'] == idn), 'meaneduc'] = mean_educ
df.loc[(df['SQBmeaned'].isnull())&(df['idhogar'] == idn), 'SQBmeaned'] = mean_educ**2
sub = sample_submission[['sig_id']].merge(test_results, on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission3.csv', index=False )<load_from_csv>
meaneduc_correction(meaneduc_null, train, h_ids )
sub1 = pd.read_csv('submission.csv')
 sub2 = pd.read_csv('submission2.csv')
 sub3 = pd.read_csv('submission3.csv')
 sub_id = sub1[['sig_id']]
sub1 = sub1[sub1.columns[sub1.columns != 'sig_id']]
sub2 = sub2[sub2.columns[sub2.columns != 'sig_id']]
sub3 = sub3[sub3.columns[sub3.columns != 'sig_id']]
sub4 =(sub1 + sub2 + sub3)/ 3
sub_mean = pd.concat([sub_id, sub4], axis=1)
 sub_mean.to_csv('submission.csv', index=False)
 sub_mean.head()<define_variables>
null_counts = train.isnull().sum()
null_counts[null_counts>0]
NB = '101'
IS_TRAIN = False
MODEL_DIR = ".. /input/503-203-tabnet-with-nonscored-features-train/model"
INT_DIR = "interim"
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0<load_from_csv>
meaneduc_correction(test_meaneduc_null, test, h_ids )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<categorify>
test_null_counts = test.isnull().sum()
test_null_counts[test_null_counts>0]
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = pd.concat([train_features, test_features])[col].values.reshape(vec_len+vec_len_test, 1)
 if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 train_features[col] = transformer.transform(train_features[col].values.reshape(vec_len, 1)).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<categorify>
train['Id'].unique().size, test['Id'].unique().size
n_comp = 50
n_dim = 15
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=DEFAULT_SEED ).fit(train_features[GENES])
 umap = UMAP(n_components=n_dim, random_state=DEFAULT_SEED ).fit(train_features[GENES])
 pd.to_pickle(pca, f"{MODEL_DIR}/{NB}_pca_g.pkl")
 pd.to_pickle(umap, f"{MODEL_DIR}/{NB}_umap_g.pkl")
 else:
pca = pd.read_pickle(f"{MODEL_DIR}/{NB}_pca_g.pkl")
 umap = pd.read_pickle(f"{MODEL_DIR}/{NB}_umap_g.pkl")
 data2 = pca.transform(data[GENES])
 data3 = umap.transform(data[GENES])
 train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train3 = data3[:train_features.shape[0]]
test3 = data3[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train3 = pd.DataFrame(train3, columns=[f'umap_G-{i}' for i in range(n_dim)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test3 = pd.DataFrame(test3, columns=[f'umap_G-{i}' for i in range(n_dim)])
 train_features = pd.concat(( train_features, train2, train3), axis=1)
 test_features = pd.concat(( test_features, test2, test3), axis=1)
 n_comp = 15
n_dim = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=DEFAULT_SEED ).fit(train_features[CELLS])
 umap = UMAP(n_components=n_dim, random_state=DEFAULT_SEED ).fit(train_features[CELLS])
 pd.to_pickle(pca, f"{MODEL_DIR}/{NB}_pca_c.pkl")
 pd.to_pickle(umap, f"{MODEL_DIR}/{NB}_umap_c.pkl")
 else:
pca = pd.read_pickle(f"{MODEL_DIR}/{NB}_pca_c.pkl")
 umap = pd.read_pickle(f"{MODEL_DIR}/{NB}_umap_c.pkl")
 data2 = pca.transform(data[CELLS])
 data3 = umap.transform(data[CELLS])
 train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train3 = data3[:train_features.shape[0]]
test3 = data3[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train3 = pd.DataFrame(train3, columns=[f'umap_C-{i}' for i in range(n_dim)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test3 = pd.DataFrame(test3, columns=[f'umap_C-{i}' for i in range(n_dim)])
 train_features = pd.concat(( train_features, train2, train3), axis=1)
 test_features = pd.concat(( test_features, test2, test3), axis=1)
<categorify>
cols = ['v2a1', 'hacdor', 'rooms', 'hacapo', 'v14a', 'refrig', 'v18q1',
'r4h3', 'r4m3', 'r4t3', 'tamhog', 'tamviv', 'hhsize', 'paredblolad',
'paredzocalo', 'paredpreb', 'pareddes', 'paredmad', 'paredzinc',
'paredfibras', 'paredother', 'pisomoscer', 'pisocemento',
'pisoother', 'pisonatur', 'pisonotiene', 'pisomadera', 'techozinc',
'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano', 'public',
'planpri', 'noelec', 'coopele', 'sanitario1', 'sanitario2',
'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5',
'elimbasu6', 'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2',
'etecho3', 'eviv1', 'eviv2', 'eviv3', 'hogar_nin', 'hogar_adul',
'hogar_mayor', 'hogar_total', 'dependency', 'meaneduc', 'bedrooms',
'overcrowding', 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4',
'tipovivi5', 'computer', 'television', 'qmobilephone', 'lugar1',
'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'Target']
if IS_TRAIN:
var_thresh = VarianceThreshold(threshold=0.5 ).fit(train_features.iloc[:, 4:])
 pd.to_pickle(var_thresh, f"{MODEL_DIR}/{NB}_variance_thresh0_5.pkl")
 else:
var_thresh = pd.read_pickle(f"{MODEL_DIR}/{NB}_variance_thresh0_5.pkl")
 data = train_features.append(test_features)
 data_transformed = var_thresh.transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 print(train_features.shape)
 print(test_features.shape )<drop_column>
warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 def check_for_wrong_data(data, columns, labelE, gpby='idhogar'):

 id_head_zero = []
idhogarId_f = []
cols_f = []
mem_f = []
grouped = data.groupby(gpby, sort=True)
 for gid in range(len(grouped)) :
members = grouped.get_group(gid)
 h_Head = members.loc[(members['parentesco1'] == 1)]
if h_Head.shape[0] == 0:
id_head_zero.append(members['idhogar'].values[0])
 continue
idhogarId_w = []
cols_t = []
mem_t = []
if members.shape[0] > 1:
for col in columns:
for m in members.iterrows() :
if h_Head[col].values[0] != m[1][col]:
if h_Head['idhogar'].values[0] not in idhogarId_w : idhogarId_w.append(h_Head['idhogar'].values[0])
 if col not in cols_t : cols_t.append(col)
 if m[1]['Id'] not in mem_t : mem_t.append(m[1]['Id'])
 data.loc[(train['Id'] == m[1]['Id']), col] = h_Head[col].values[0]
idhogarId_f.append(idhogarId_w); cols_f.append(cols_t); mem_f.append(mem_t)
 if len(idhogarId_w)> 0:
for i in range(len(idhogarId_w)) :
print("Household with Id: "
+str(labelE.inverse_transform([idhogarId_w[i]])[0])
 +" has " + str(len(mem_t)) + " member(s)with diff.value(s)of " + str(len(cols_t)) + " column(s )."
+ " " + str(cols_t))
return id_head_zero, idhogarId_f, cols_f, mem_f
train = train_features[train_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<load_pretrained>
id_head_zero, *_ = check_for_wrong_data(train, cols, lb2 )
train.to_pickle(f"{INT_DIR}/{NB}_train_preprocessed.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_preprocessed.pkl" )<init_hyperparams>
train.loc[(train['idhogar'] == id_head_zero[11])]
NB = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
HIDDEN_SIZE = 2048
EPOCHS = 15
BATCH_SIZE = 2048
NFOLDS = 10
NREPEATS = 1
NSEEDS = 5
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2<categorify>
id_head_zero, *_ = check_for_wrong_data(test, cols, lb4 )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def calc_valid_log_loss(train, target, target_cols):
y_pred = train[target_cols].values
y_true = target[target_cols].values
y_true_t = torch.from_numpy(y_true.astype(np.float64)).clone()
y_pred_t = torch.from_numpy(y_pred.astype(np.float64)).clone()
return torch.nn.BCELoss()(y_pred_t, y_true_t ).to('cpu' ).detach().numpy().copy()<normalization>
possible_outliers = [columns[0]] + [columns[98]]; columns[0], columns[98]
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=HIDDEN_SIZE):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<prepare_x_and_y>
train.loc[(train['v2a1'] > 300000), ['idhogar', 'v2a1', 'Target']].query("Target != 4" )
def run_training(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=PCT_START, div_factor=DIV_FACS,
max_lr=MAX_LR, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
best_loss_epoch = -1
if IS_TRAIN:
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 if valid_loss < best_loss:
best_loss = valid_loss
best_loss_epoch = epoch
oof[val_idx] = valid_preds
model.to('cpu')
 torch.save(model.state_dict() , f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth")
 model.to(DEVICE)
 if epoch % 10 == 0 or epoch == EPOCHS-1:
print(f"seed: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}, best_loss: {best_loss:.6f}, best_loss_epoch: {best_loss_epoch}")
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.load_state_dict(torch.load(f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth"))
model.to(DEVICE)
 if not IS_TRAIN:
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<split>
train.loc[(train['v2a1'] > 2000000), ['idhogar', 'v2a1', 'Target']]
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = RepeatedMultilabelStratifiedKFold(n_splits=NFOLDS, n_repeats=NREPEATS, random_state=None)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions<feature_engineering>
train.loc[(train['meaneduc'] > 25), ['idhogar', 'Target', 'meaneduc']].query("Target != 4" )
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions<load_from_csv>
columns = ['v2a1', 'rooms', 'tamhog', 'overcrowding', 'v18q1', 'r4t3', 'meaneduc', 'qmobilephone', 'Target']
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<load_pretrained>
train.drop(['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding',
'SQBdependency', 'SQBmeaned', 'agesq'], axis = 1, inplace=True )
train = pd.read_pickle(f"{INT_DIR}/101_train_preprocessed.pkl")
 test = pd.read_pickle(f"{INT_DIR}/101_test_preprocessed.pkl" )<load_pretrained>
test.drop(['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding',
'SQBdependency', 'SQBmeaned', 'agesq'], axis = 1, inplace=True )
train_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_train_preprocessed.pkl")
 test_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_test_preprocessed.pkl" )<merge>
DropCols = ['energcocinar1', 'energcocinar4', 'energcocinar3', 'energcocinar2', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']
train['CookingType'] = np.argmax(np.array(train[[ 'energcocinar1', 'energcocinar4', 'energcocinar3', 'energcocinar2' ]]), axis=1)
 train['WallType'] = np.argmax(np.array(train[[ 'epared1', 'epared2', 'epared3' ]]), axis=1)
 train['RoofType'] = np.argmax(np.array(train[[ 'etecho1', 'etecho2', 'etecho3' ]]), axis=1)
 train['FloorType'] = np.argmax(np.array(train[[ 'eviv1', 'eviv2', 'eviv3' ]]), axis=1)
 train['HouseType'] = np.argmax(np.array(train[[ 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5' ]]), axis=1)
 test['CookingType'] = np.argmax(np.array(test[[ 'energcocinar1', 'energcocinar4', 'energcocinar3', 'energcocinar2' ]]), axis=1)
 test['WallType'] = np.argmax(np.array(test[[ 'epared1', 'epared2', 'epared3' ]]), axis=1)
 test['RoofType'] = np.argmax(np.array(test[[ 'etecho1', 'etecho2', 'etecho3' ]]), axis=1)
 test['FloorType'] = np.argmax(np.array(test[[ 'eviv1', 'eviv2', 'eviv3' ]]), axis=1)
 test['HouseType'] = np.argmax(np.array(test[[ 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5' ]]), axis=1)
 train.drop(DropCols, axis=1, inplace=True)
 test.drop(DropCols, axis=1, inplace=True)
 test.shape, train.shape
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0]
print(train_targets_nonscored.shape)
 train = train.merge(train_targets_nonscored, on='sig_id' )<compute_test_metric>
train['phones-per-mem'] = train['qmobilephone'] / train['tamviv']
train['tablets-per-mem'] = train['v18q1'] / train['tamviv']
train['rooms-per-mem'] = train['rooms'] / train['tamviv']
train['rent-per-adult'] = train['v2a1'] / train['hogar_adul']
test['phones-per-mem'] = test['qmobilephone'] / test['tamviv']
test['tablets-per-mem'] = test['v18q1'] / test['tamviv']
test['rooms-per-mem'] = test['rooms'] / test['tamviv']
test['rent-per-adult'] = test['v2a1'] / test['hogar_adul']
run_seeds(train, test, feature_cols, target_cols )<compute_test_metric>
to_drop = chk_n_remove_corr(train)
 to_drop
valid_loss_total = calc_valid_log_loss(train, target, target_cols)
 print(f"CV loss: {valid_loss_total}" )<load_pretrained>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1','tamviv','hogar_nin',
'CookingType', 'WallType', 'RoofType', 'HouseType' , 'FloorType',
'hogar_adul','hogar_mayor', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding',
'phones-per-mem', 'tablets-per-mem', 'rooms-per-mem', 'rent-per-adult']
ind_bool = ['v18q', 'dis', 'male', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'mobilephone']
ind_ordered = ['age', 'escolari', 'rez_esc']
train.to_pickle(f"{INT_DIR}/{NB}_train_nonscored_pred.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_nonscored_pred.pkl" )<compute_test_metric>
train[hh_bool + ind_bool] = train[hh_bool + ind_bool].astype(bool)
 test[hh_bool + ind_bool] = test[hh_bool + ind_bool].astype(bool )
valid_results = train_targets_nonscored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_nonscored[target_cols].values
y_true = y_true > 0.5
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<install_modules>
train[hh_cont] = train[hh_cont].astype('float64')
 test[hh_cont] = test[hh_cont].astype('float64');
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<import_modules>
train[hh_ordered + ind_ordered] = train[hh_ordered + ind_ordered].astype(int)
 test[hh_ordered + ind_ordered] = test[hh_ordered + ind_ordered].astype(int);
from pytorch_tabnet.tab_model import TabNetRegressor<set_options>
train['Target'] = train['Target'].astype(int);
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<define_variables>
train.isnull().sum() [train.isnull().sum() > 0], test.isnull().sum() [test.isnull().sum() > 0]
NB = '503'
NB_PREV = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
NFOLDS = 10
NREPEATS = 1
NSEEDS = 3
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2<load_from_csv>
train.loc[train['rent-per-adult'].isnull() , 'rent-per-adult'] = train.loc[train['rent-per-adult'].isnull() , 'v2a1'].values[0]
test.loc[test['rent-per-adult'].isnull() , 'rent-per-adult'] = test.loc[test['rent-per-adult'].isnull() , 'v2a1'].values[0]
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
train.isnull().sum() [train.isnull().sum() > 0], test.isnull().sum() [test.isnull().sum() > 0]
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
for c in train.columns:
if train[c].dtype != 'float64': continue
s = np.where(train[c].values >= np.finfo(np.float32 ).max)
 if len(s[0])>0:
print(c)
 print(s )
use_test_for_preprocessing = False
for col in(GENES + CELLS):
if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 if use_test_for_preprocessing:
raw_vec = pd.concat([train_features, test_features])[col].values.reshape(vec_len+vec_len_test, 1)
 transformer.fit(raw_vec)
 else:
raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 train_features[col] = transformer.transform(train_features[col].values.reshape(vec_len, 1)).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<categorify>
train[train['rent-per-adult'] > np.finfo(np.float32 ).max][['rent-per-adult']]
n_comp = 90
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[GENES])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 data2 =(fa.transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[CELLS])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 data2 =(fa.transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<train_on_grid>
train[train['rent-per-adult'] > np.finfo(np.float32 ).max][['Id', 'idhogar', 'v2a1', 'hogar_adul', 'age']]
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 if IS_TRAIN:
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 pd.to_pickle(kmeans, f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 else:
kmeans = pd.read_pickle(f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 train[f'clusters_{kind}'] = kmeans.predict(train_)
 test[f'clusters_{kind}'] = kmeans.predict(test_)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )<compute_test_metric>
train[train['idhogar']==1959][['idhogar', 'v2a1', 'age']]
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )<filter>
train[train['idhogar']==2908][['idhogar', 'v2a1', 'age']]
remove_vehicle = True
if remove_vehicle:
trt_idx = train_features['cp_type']=='trt_cp'
train_features = train_features.loc[trt_idx].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[trt_idx].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[trt_idx].reset_index(drop=True)
 else:
pass<drop_column>
for h_id in h_ids:
rent_per_adul = train.loc[(train['idhogar']==h_id), 'v2a1'].values[0] / train.loc[(train['idhogar']==h_id)].shape[0]
train.loc[train['idhogar']==h_id, 'rent-per-adult'] = rent_per_adul
train.loc[train['idhogar']==h_id, 'rent-per-adult_sum'] = train.loc[(train['idhogar']==h_id), 'v2a1'].values[0]
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<load_pretrained>
for c in test.columns:
if test[c].dtype != 'float64': continue
s = np.where(test[c].values >= np.finfo(np.float32 ).max)
 if len(s[0])>0:
print(c)
 print(s )
train_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_train_nonscored_pred.pkl')
 test_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_test_nonscored_pred.pkl' )<filter>
test[test['rent-per-adult'] > np.finfo(np.float32 ).max][['rent-per-adult']]
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0] 
<merge>
test[test['rent-per-adult'] > np.finfo(np.float32 ).max][['Id', 'idhogar', 'v2a1', 'hogar_adul', 'age']]
train = train.merge(train_nonscored_pred[train_targets_nonscored.columns], on='sig_id')
 test = test.merge(test_nonscored_pred[train_targets_nonscored.columns], on='sig_id' )<categorify>
h_ids = test[test['rent-per-adult'] > np.finfo(np.float32 ).max]['idhogar'].unique()
for h_id in h_ids:
rent_per_adul = test.loc[(test['idhogar']==h_id), 'v2a1'].values[0] / test.loc[(test['idhogar']==h_id)].shape[0]
test.loc[test['idhogar']==h_id, 'rent-per-adult'] = rent_per_adul
test.loc[test['idhogar']==h_id, 'rent-per-adult_sum'] = test.loc[(test['idhogar']==h_id), 'v2a1'].values[0]
nonscored_target = [c for c in train_targets_nonscored.columns if c != "sig_id"]
for col in(nonscored_target):
vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<define_variables>
def make_new_features_grouping(df, dtypes, gpby, customAggFunc=None):

 if 'Target' in df.columns: numeric_type = df.select_dtypes(dtypes ).drop(['Target', 'Id'], axis=1 ).copy()
else: numeric_type = df.select_dtypes(dtypes ).drop(['Id'], axis=1 ).copy()
funcs = ['count', 'mean', 'max', 'min', 'sum', 'std', 'var', 'quantile']
if customAggFunc is None: new = numeric_type.groupby(gpby ).agg(funcs)
 elif isinstance(customAggFunc, list): new = numeric_type.groupby(gpby ).agg(funcs + customAggFunc)
 else: new = numeric_type.groupby(gpby ).agg(funcs + [customAggFunc])
 columns = []
for old_col in new.columns.levels[0]:
if old_col != 'idhogar':
for new_col in new.columns.levels[1]:
columns.append(old_col + '_' + new_col)
 new.columns = columns
return df.merge(new.reset_index() , on="idhogar", how='left' )
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
len(feature_cols )<define_variables>
%time train = make_new_features_grouping(train, ['number', bool], "idhogar")
 %time test = make_new_features_grouping(test, ["number", bool], "idhogar" )
num_features=len(feature_cols)
 num_targets=len(target_cols )<normalization>
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True )
class LabelSmoothing(nn.Module):

 def __init__(self, smoothing=0.0, n_cls=2):

 super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing + smoothing / n_cls
self.smoothing = smoothing / n_cls
def forward(self, x, target):
probs = torch.nn.functional.sigmoid(x,)
 target1 = self.confidence * target +(1-target)* self.smoothing
loss = -(torch.log(probs+1e-15)* target1 +(1-target1)* torch.log(1-probs+1e-15))
return loss.mean()
class SmoothedLogLossMetric(Metric):

 def __init__(self, smoothing=0.001):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0)
 class LogLossMetric(Metric):

 def __init__(self, smoothing=0.0):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0 )<categorify>
scaler1 = RobustScaler()
scaler2 = RobustScaler()
scaled1 = scaler1.fit_transform(train.drop(['Target', 'Id', 'idhogar'], axis=1))
scaled2 = scaler2.fit_transform(test.drop(['Id', 'idhogar'], axis=1))
cols1 = train.drop(['Target', 'Id', 'idhogar'], axis=1 ).columns
cols2 = test.drop(['Id', 'idhogar'], axis=1 ).columns
trPCA = pd.DataFrame(scaled1, index=np.arange(train.shape[0]), columns=cols1)
 tsPCA = pd.DataFrame(scaled2, index=np.arange(test.shape[0]), columns=cols2 )
def process_data(data):
data.loc[:, 'cp_time'] = data.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2, 0: 0, 1: 1, 2: 2})
 data.loc[:, 'cp_dose'] = data.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1, 0: 0, 1: 1})
 return data
def run_training_tabnet(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed, filename="tabnet"):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 loss_fn = LabelSmoothing(0.001)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
if IS_TRAIN:
model.fit(X_train=x_train, y_train=y_train,
eval_set=[(x_valid, y_valid)], eval_metric=[LogLossMetric, SmoothedLogLossMetric],
max_epochs=200, patience=50, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=loss_fn 
)
 model.save_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}")
 x_test = test_[feature_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 model.load_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}.model")
 valid_preds = model.predict(x_valid)
 valid_preds = torch.sigmoid(torch.as_tensor(valid_preds)).detach().cpu().numpy()
oof[val_idx] = valid_preds
predictions = model.predict(x_test)
 predictions = torch.sigmoid(torch.as_tensor(predictions)).detach().cpu().numpy()
return oof, predictions<split>
pca = PCA(n_components=5, svd_solver='full' )
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state = seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training_tabnet(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions<load_pretrained>
transformed1 = pca.fit_transform(trPCA)
 transformed2 = pca.transform(tsPCA)
 for i in range(5):
train[f'PCA{i+1}'] = transformed1[:,i]
test[f'PCA{i+1}'] = transformed2[:,i]
train.shape, test.shape
train.to_pickle(f"{INT_DIR}/{NB}_pre_train.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_pre_test.pkl" )<compute_test_metric>
pca.explained_variance_ratio_
run_seeds(train, test, feature_cols, target_cols, NFOLDS, NSEEDS )<load_pretrained>
train['Target'].value_counts() , 774+1221+1558+1500
train.to_pickle(f"{INT_DIR}/{NB}_train.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test.pkl" )<compute_test_metric>
def train_val_split(rows, tvlen=None, vper=None):

 if tvlen is not None and vper is None:
count1 = tvlen[0]
count2 = tvlen[1]
elif tvlen is None and vper is not None:
c = rows.sum()
count1 = int(( 1-vper)*c)
 count2 = int(vper*c)
 else:
raise Exception('One of "tvlen" or "vper" should be given.')
 rowst, rowsv = rows.copy() , rows.copy()
for i in range(len(rows)) :
if not count1:
rowst[i] = False
if not count2:
rowsv[i] = False
count2 -= rowsv[i]
continue
rowsv[i] = False
count1 -= rowst[i]
return rowst, rowsv
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_true = y_true > 0.5
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
rows123t, rows123v = train_val_split(rows123, vper=0.1)
 rows4t, rows4v = train_val_split(rows4, tvlen=(1300, 200))
rows123t.sum() , rows123v.sum() , rows4t.sum() , rows4v.sum()
sub6 = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub6.to_csv('submission.csv', index=False )<import_modules>
train.drop(['Id', 'idhogar'], axis=1, inplace=True )
import glob 
<load_from_csv>
xtrain, xvalid = train.loc[rows123t|rows4t].drop('Target', axis=1 ).copy() , train.loc[rows123v|rows4v].drop('Target', axis=1 ).copy()
ytrain, yvalid = train['Target'].loc[rows123t|rows4t].copy() , train['Target'].loc[rows123v|rows4v].copy()
!python.. /input/inference-best-lb/deepinsight-resnest-lightning-v2-inference.py
sub5 = pd.read_csv('submission_resnest_v2.csv' )<load_from_csv>
ytrain.value_counts()
!python.. /input/inference-best-lb/deepinsight-efficientnet-lightning-v7-b3-inference.py
sub4 = pd.read_csv('./submission_effnet_v7_b3.csv' )<load_from_csv>
yvalid.value_counts()
! python.. /input/inference-best-lb/simple-nn-new-split-inference.py
sub3 = pd.read_csv('./submission.csv' )<load_from_csv>
train['Target'].value_counts()
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 !python.. /input/inference-best-lb/2heads-ResNest-inference.py
sub2 = pd.read_csv('./submission.csv' )<load_from_csv>
target1 = train.loc[train['Target']==1].copy()
target2 = train.loc[train['Target']==2].copy()
target3 = train.loc[train['Target']==3].copy()
!python.. /input/inference-best-lb/3stage-nn-inference.py
sub1 = pd.read_csv('./submission_2stageNN_with_ns_oldcv_0.01822.csv' )<load_from_csv>
target1 = pd.concat([target1]*8, ignore_index=True ).copy() ; target1.shape
!python.. /input/inference-best-lb/simple-nn-old-split-inference.py
sub7 = pd.read_csv('submission_script_simpleNN_oldcv_0.01836.csv' )<save_to_csv>
target2 = pd.concat([target2]*4, ignore_index=True ).copy() ; target2.shape
submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission.iloc[:, 1:] = 0
submission.iloc[:, 1:] =(sub1.iloc[:,1:]*0.37 + sub3.iloc[:,1:]*0.1 + sub4.iloc[:,1:]*0.18 +sub5.iloc[:,1:]*0.15)*0.9 + sub6.iloc[:,1:]*0.1 + sub7.iloc[:,1:]*0.09 + sub2.iloc[:,1:]*0.09
submission.to_csv('submission.csv', index=False )<install_modules>
target3 = pd.concat([target3]*5, ignore_index=True ).copy() ; target3.shape
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
!mkdir -p /root/.cache/torch/hub/checkpoints/
!cp.. /input/gen-efficientnet-pretrained/tf_efficientnet_*.pth /root/.cache/torch/hub/checkpoints/
!ls -la /root/.cache/torch/hub/checkpoints/
!pip install.. /input/gen-efficientnet-pytorch<load_from_csv>
train2 = train.copy()
train2 = pd.concat([train2, target1, target2, target3], ignore_index=True); train2.shape
def create_submission() :
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_target_columns = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).columns
submission = pd.DataFrame(columns=train_target_columns)
 submission.sig_id = test.sig_id
submission.iloc[:, 1:] = 0.0
return submission
def load_df_list(file_paths):
df_list = []
for p in file_paths:
if type(p)is pd.DataFrame:
df_list.append(p)
 else:
df_list.append(pd.read_csv(p))
df1 = df_list[0]
for(i, df2)in enumerate(df_list[1:]):
assert len(df1)== len(df2)
 df_list[i + 1] = df2.set_index('sig_id' ).loc[df1.sig_id].reset_index()
return df_list
def apply_emsamble(df_list, weights=None):
return np.average([df.iloc[:, 1:].values for df in df_list], axis=0, weights=weights)
 def create_ensamble_submission(paths, weights=None):
submission = create_submission()
df_list = load_df_list(paths)
 ensamble = apply_emsamble(df_list, weights=weights)
 submission.iloc[:, 1:] = ensamble
print('Check NA: ', submission.isna().any().any())
 return submission<save_to_csv>
xtrain2, xvalid2 = train2.iloc[:15000].drop(['Target'], axis=1 ).copy() , train2.iloc[15000:].drop(['Target'], axis=1 ).copy()
ytrain2, yvalid2 = train2.iloc[:15000]['Target'].copy() , train2.iloc[15000:]['Target'].copy()
submission = create_ensamble_submission([
'blending-02-tabnet-submission.csv',
'blending-01-nn-0-submission.csv',
'blending-01-nn-1-submission.csv',
'blending-01-nn-2-submission.csv',
'blending-01-nn-3-submission.csv',
'blending-01-nn-4-submission.csv',
'blending-01-nn-5-submission.csv',
'deepinsignt-submission_0.csv',
'deepinsignt-submission_1.csv',
], weights=(0.5, 0.7, 0.7, 0.5, 1.5, 1.5, 0.5, 0.4, 0.5))
submission.to_csv('submission.csv', index=False)
 pd.read_csv('submission.csv' ).head()<define_variables>
from sklearn.ensemble import RandomForestClassifier
import math
LABEL_SMOOTHING=0.001
SEEDS = [120,2524]
SPLITS = 7
BATCH_SIZE = 64
EPOCHS = 60
VARIANCE_THRESHOLD = True
ADD_NON_TRAIN = True
ADD_SEEDS = SEEDS
ADD_SPLITS = SPLITS
ADD_BATCH_SIZE = BATCH_SIZE
ADD_EPOCHS = EPOCHS
RUN_SNN = True
SNN_SEEDS = SEEDS + [42]
SNN_SPLITS = SPLITS
SNN_BATCH_SIZE = BATCH_SIZE
SNN_EPOCHS = EPOCHS
RUN_NN = True
NN_SEEDS = SEEDS
NN_SPLITS = SPLITS
NN_BATCH_SIZE = BATCH_SIZE
NN_EPOCHS = EPOCHS
RUN_NN2 = True
NN2_SEEDS = SEEDS
NN2_SPLITS = SPLITS
NN2_BATCH_SIZE = BATCH_SIZE
NN2_EPOCHS = EPOCHS
RUN_RNN = True
RNN_SEEDS = SEEDS
RNN_SPLITS = SPLITS
RNN_BATCH_SIZE = 128
RNN_EPOCHS = EPOCHS
RUN_TABNET = True
TAB_SEEDS = SEEDS
TAB_SPLITS = SPLITS
TAB_BATCH_SIZE = BATCH_SIZE
TAB_EPOCHS = EPOCHS
RUN_STACKING = True
STK_SEEDS = SEEDS + [42]
STK_SPLITS = SPLITS
STK_EPOCHS = EPOCHS
STK_BATCH_SIZE = BATCH_SIZE
RE_RUN_NN = True
RE_NN_SEEDS = SEEDS<import_modules>
def print_score(m, trn, val):

 print("Train F1score: ", str(F1score(trn[1], m.predict(trn[0]))),
", Valid.F1score: ", str(F1score(val[1], m.predict(val[0]))))
print("Train Acc.: ", str(m.score(trn[0], trn[1])) ,
", Valid.Acc.: ", str(m.score(val[0], val[1])) )
import pandas as pd
import numpy as np
import random
import os
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
from tensorflow_addons.layers import WeightNormalization
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping<feature_engineering>
m = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.7, n_jobs=-1)
 m.fit(xtrain, ytrain)
 print_score(m,(xtrain, ytrain),(xvalid, yvalid))
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 torch.manual_seed(seed)
 session_conf = tf.compat.v1.ConfigProto(
 intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess)
 def swish(x):
return x * K.sigmoid(x)
 def show_metrics(valid_preds, show_each=True):
metrics = []
for _target in valid_preds.columns:
logloss = log_loss(train_targets_scored.iloc[:,1:].loc[:, _target], valid_preds.loc[:, _target])
 metrics.append(logloss)
 if show_each:
print(f'column: {_target}, log loss: {logloss}')
 print(f'OOF Metric: {np.mean(metrics)}')
 return metrics<load_from_csv>
m2 = RandomForestClassifier(n_estimators=100, min_samples_leaf=150, max_features=0.5, n_jobs=-1)
 m2.fit(xtrain2, ytrain2)
 print_score(m2,(xtrain2, ytrain2),(xvalid2, yvalid2))
train_features = pd.read_csv(f'.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv(f'.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(f'.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv(f'.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv(f'.. /input/lish-moa/sample_submission.csv' )<define_variables>
a, b, c, d = train_test_split(train.drop('Target', axis=1), train['Target'], test_size=0.20,
stratify=train['Target'])
 xtrain3, xvalid3, ytrain3, yvalid3 = a.copy() , b.copy() , c.copy() , d.copy()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<prepare_x_and_y>
m3 = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1, class_weight='balanced')
 m3.fit(xtrain3, ytrain3)
 print_score(m3,(xtrain3, ytrain3),(xvalid3, yvalid3))
_X_train = train_features.copy()
_X_test = test_features.copy()
_y_train = train_targets_scored.copy()<categorify>
fi = pd.DataFrame({'cols':xtrain.columns, 'imp':m.feature_importances_} ).sort_values('imp', ascending=False); fi[0:10]
def deal_with_cp_type(_X_train, _X_test):
train_trt_index = _X_train.cp_type == 'trt_cp'
test_trt_index = _X_test.cp_type == 'trt_cp'
_X_train = _X_train[train_trt_index]
_X_test = _X_test
del _X_train['cp_type']
del _X_test['cp_type']
return _X_train, _X_test, train_trt_index, test_trt_index
_X_train, _X_test, train_trt_index, test_trt_index = deal_with_cp_type(_X_train, _X_test)
 def deal_with_cp_dose(_X_train, _X_test):
_X_train = pd.concat([_X_train, pd.get_dummies(_X_train.cp_dose)], axis=1)
 del _X_train['cp_dose']
_X_test = pd.concat([_X_test, pd.get_dummies(_X_test.cp_dose)], axis=1)
 del _X_test['cp_dose']
return _X_train, _X_test
_X_train, _X_test = deal_with_cp_dose(_X_train, _X_test)
 def deal_with_sig_id(_X_train, _X_test):
del _X_train['sig_id']
del _X_test['sig_id']
return _X_train, _X_test
_X_train, _X_test = deal_with_sig_id(_X_train, _X_test)
 def deal_with_y(_y_train, train_trt_index):
_y_train = _y_train[train_trt_index]
del _y_train['sig_id']
return _y_train
_y_train = deal_with_y(_y_train, train_trt_index)
 BASE_COLS = ['cp_time', 'D1', 'D2']
_X_train_dae = _X_train.copy()
_X_test_dae = _X_test.copy()<define_variables>
to_keep = fi.loc[(fi['imp']>0.005), 'cols']; to_keep.shape
_X_train = _X_train.reset_index()<prepare_x_and_y>
xtrain_new, xvalid_new = xtrain[to_keep].copy() , xvalid[to_keep].copy()
non_scored_ones = pd.DataFrame()
non_scored_ones['col_name'] = ''
non_scored_ones['item_counts'] = 0
for col in train_targets_nonscored.columns[1:]:
item_counts = len(train_targets_nonscored[train_targets_nonscored[col] == 1])
 non_scored_ones = non_scored_ones.append({'col_name':col, 'item_counts':item_counts}, ignore_index=True)
 non_scored_target_cols = non_scored_ones[non_scored_ones.item_counts > 10].col_name
y_non_train = train_targets_nonscored[non_scored_target_cols]
y_non_train = y_non_train[train_trt_index].reset_index(drop=True )<categorify>
m = RandomForestClassifier(n_estimators=100, min_samples_leaf=3, max_features=0.5, n_jobs=-1)
 m.fit(xtrain_new, ytrain)
 print_score(m,(xtrain_new, ytrain),(xvalid_new, yvalid))
def yield_pca(_X_train, _X_test, prefix, decomp_cols, comp=50, random_state=42):
pca = PCA(n_components=comp, random_state=random_state)
 pca.fit(pd.concat([_X_train.loc[:,decomp_cols], _X_test.loc[:,decomp_cols]]))
_X_train_PCA = pca.transform(_X_train[decomp_cols])
 _X_test_PCA = pca.transform(_X_test[decomp_cols])
 _X_train_PCA = pd.DataFrame(_X_train_PCA, columns=[f'pca_{prefix}-{i}' for i in range(comp)])
 _X_test_PCA = pd.DataFrame(_X_test_PCA, columns=[f'pca_{prefix}-{i}' for i in range(comp)])
 return _X_train_PCA, _X_test_PCA
_X_train_G_PCA, _X_test_G_PCA = yield_pca(_X_train, _X_test, 'G', GENES, 90)
 _X_train_C_PCA, _X_test_C_PCA = yield_pca(_X_train, _X_test, 'C', CELLS, 27)
 _X_train_G_PCA_Dense, _X_test_G_PCA_Dense = yield_pca(_X_train, _X_test, 'Gd', GENES, 8)
 _X_train_C_PCA_Dense, _X_test_C_PCA_Dense = yield_pca(_X_train, _X_test, 'Cd', CELLS, 1)
 _X_train = pd.concat([_X_train, _X_train_G_PCA], axis=1)
 _X_train = pd.concat([_X_train, _X_train_C_PCA], axis=1)
 _X_train = pd.concat([_X_train, _X_train_G_PCA_Dense], axis=1)
 _X_train = pd.concat([_X_train, _X_train_C_PCA_Dense], axis=1)
 _X_test = pd.concat([_X_test, _X_test_G_PCA], axis=1)
 _X_test = pd.concat([_X_test, _X_test_C_PCA], axis=1)
 _X_test = pd.concat([_X_test, _X_test_G_PCA_Dense], axis=1)
 _X_test = pd.concat([_X_test, _X_test_C_PCA_Dense], axis=1)
 PCA_G = [col for col in _X_train.columns if col.startswith('pca_G-')] + [col for col in _X_train.columns if col.startswith('pca_Gd-')]
PCA_C = [col for col in _X_train.columns if col.startswith('pca_C-')] + [col for col in _X_train.columns if col.startswith('pca_Cd-')]<create_dataframe>
fi = pd.DataFrame({'cols':xtrain2.columns, 'imp':m2.feature_importances_} ).sort_values('imp', ascending=False); fi[0:10]
def make_feature(data, target_feats, create, prefix):
target_data = data.loc[:,target_feats]
invoker = getattr(target_data, create)
 return pd.DataFrame(invoker(axis=1), columns=[f'{prefix}_{create}'])
 for method in ['sum', 'mean', 'std', 'skew', 'kurt', 'median']:
_X_train = pd.concat([_X_train, make_feature(_X_train, GENES+CELLS, method, 'gce')], axis = 1)
 _X_train = pd.concat([_X_train, make_feature(_X_train, GENES, method, 'ge')], axis = 1)
 _X_test = pd.concat([_X_test, make_feature(_X_test, GENES+CELLS, method, 'gce')], axis = 1)
 _X_test = pd.concat([_X_test, make_feature(_X_test, GENES, method, 'ge')], axis = 1)
 GC_EX = [col for col in _X_train.columns if col.startswith('gce_')]
G_EX = [col for col in _X_train.columns if col.startswith('ge_')]<categorify>
to_keep = fi.loc[(fi['imp']>0.005), 'cols']; to_keep.shape
train_col_names = _X_train.columns
test_col_names = _X_test.columns
TRANSFORM_TARGET_COLS = BASE_COLS + GENES + CELLS + PCA_G + PCA_C + G_EX + GC_EX
def rank_gauss(_X_train, _X_test, cols=TRANSFORM_TARGET_COLS, random_state=42):
qt = QuantileTransformer(random_state=random_state, output_distribution='normal')
 qt.fit(pd.concat([_X_train[cols], _X_test[cols]]))
_X_train[cols] = qt.transform(_X_train[cols])
 _X_test[cols] = qt.transform(_X_test[cols])
 return _X_train, _X_test
def standard_scaler(_X_train, _X_test, cols=TRANSFORM_TARGET_COLS):
ss = StandardScaler()
ss.fit(pd.concat([_X_train[cols], _X_test[cols]]))
_X_train[cols] = ss.transform(_X_train[cols])
 _X_test[cols] = ss.transform(_X_test[cols])
 return _X_train, _X_test
_X_train, _X_test = rank_gauss(_X_train, _X_test)
 _X_train, _X_test = standard_scaler(_X_train, _X_test)
 _X_train.columns = train_col_names
_X_test.columns = test_col_names<concatenate>
xtrain2_new, xvalid2_new = xtrain2[to_keep].copy() , xvalid2[to_keep].copy()
m2 = RandomForestClassifier(n_estimators=100, min_samples_leaf=150, max_features=0.5, n_jobs=-1)
 m2.fit(xtrain2_new, ytrain2)
 print_score(m2,(xtrain2_new, ytrain2),(xvalid2_new, yvalid2))
if VARIANCE_THRESHOLD:
vt_cols = vt_cols.values.tolist() + BASE_COLS + PCA_G + PCA_C + GC_EX + G_EX
_X_train_nl = pd.concat([_X_train.iloc[:,:1], _X_train.loc[:,vt_cols]], axis=1)
 _X_test_nl = _X_test.loc[:,vt_cols]
vt_cols_leaky = vt_cols_leaky.values.tolist() + BASE_COLS + PCA_G + PCA_C + GC_EX + G_EX
_X_train = pd.concat([_X_train.iloc[:,:1], _X_train.loc[:,vt_cols_leaky]], axis=1)
 _X_test = _X_test.loc[:,vt_cols_leaky]
GENES = [col for col in _X_train.columns if col.startswith('g-')]
CELLS = [col for col in _X_train.columns if col.startswith('c-')]<prepare_x_and_y>
fi = pd.DataFrame({'cols':xtrain3.columns, 'imp':m3.feature_importances_} ).sort_values('imp', ascending=False); fi[0:10]
_X_train = _X_train.set_index('index')
 X_train = _X_train.copy()
X_test = _X_test.copy()
_X_train_nl = _X_train_nl.set_index('index')
 X_train_nl = _X_train_nl.copy()
X_test_nl = _X_test_nl.copy()
y_train = _y_train.copy()<predict_on_test>
to_keep = fi.loc[(fi['imp']>0.005), 'cols']; to_keep.shape
def train_and_predict(name, model_func, _X_i_train, _y_i_train, _X_i_test, orig_targets, result_template,
seeds, splits, epochs, batch_size, shuffle_rows=False, pick_col_size=800,
do_show_metrics=True, show_each_metrics=True):
st = time.time()
is_list = isinstance(_X_i_train, list)
 val_result = orig_targets.copy()
val_result.loc[:, :] = 0
sub_result = result_template.copy()
sub_result.loc[:, 1:] = 0
for h, seed in enumerate(seeds):
seed_everything(seed)
 for i,(train_idx, valid_idx)in enumerate(MultilabelStratifiedKFold(n_splits=splits, random_state=seed, shuffle=True)
.split(_y_i_train, _y_i_train)) :
print(f'Fold {i+1}')
 if is_list:
_X_train = [_X_i_train[0].loc[:,:].values[train_idx], _X_i_train[1].loc[:,:].values[train_idx]]
_X_valid = [_X_i_train[0].loc[:,:].values[valid_idx], _X_i_train[1].loc[:,:].values[valid_idx]]
else:
_X_train = _X_i_train.loc[:,:].values[train_idx]
_X_valid = _X_i_train.loc[:,:].values[valid_idx]
_y_train = _y_i_train.values[train_idx]
_y_valid = _y_i_train.values[valid_idx]
if is_list:
model = model_func(len(_X_i_train[0].columns), len(_X_i_train[1].columns))
else:
model = model_func(len(_X_i_train.columns))
model.fit(_X_train, _y_train,
validation_data=(_X_valid, _y_valid),
epochs=epochs, batch_size=batch_size,
callbacks=[
ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=3, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'{name}_{seed}_{i}.hdf5', monitor = 'val_logloss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 , EarlyStopping(monitor = 'val_logloss', min_delta = 1e-4, patience = 5, mode = 'min', baseline = None, restore_best_weights = True)
 ], verbose=2)
 model.load_weights(f'{name}_{seed}_{i}.hdf5')
 val_result.iloc[_y_i_train.iloc[valid_idx,:].index.values, :] += model.predict(_X_valid)
 if is_list:
sub_result.loc[test_trt_index, sub_result.columns[1:]] += model.predict([_X_i_test[0].loc[test_trt_index, :], _X_i_test[1].loc[test_trt_index, :]])
 else:
sub_result.loc[test_trt_index, sub_result.columns[1:]] += model.predict(_X_i_test.loc[test_trt_index, :])
 print('')
 tmp_result = val_result.copy()
tmp_result.iloc[:,1:] = val_result.iloc[:,1:] /(h + 1)
 print(f' ---- seed:{seed}, ensemble:{h + 1}')
 if do_show_metrics:
_ = show_metrics(tmp_result, show_each=False)
 val_result.iloc[:,1:] = val_result.iloc[:,1:] / len(seeds)
 if do_show_metrics:
metrics = show_metrics(val_result, show_each=show_each_metrics)
 else:
metrics = None
sub_result.iloc[:, 1:] = sub_result.iloc[:, 1:] /(len(seeds)* splits)
 print(f' elapsed: {time.time() - st}')
 return sub_result, val_result, metrics<compute_test_metric>
xtrain3_new, xvalid3_new = xtrain3[to_keep].copy() , xvalid3[to_keep].copy()
m3 = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1, class_weight="balanced")
 m3.fit(xtrain3_new, ytrain3)
 print_score(m3,(xtrain3_new, ytrain3),(xvalid3_new, yvalid3))
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -K.mean(y_true*K.log(y_pred)+(1-y_true)*K.log(1-y_pred))<prepare_x_and_y>
import scipy
from scipy.cluster import hierarchy as hc
output_bias=tf.keras.initializers.Constant(-np.log(y_train.mean(axis=0 ).to_numpy()))<choose_model_class>
m = RandomForestClassifier(n_estimators=50, min_samples_leaf=25, max_features=0.6, n_jobs=-1, oob_score=True)
 m.fit(xtrain_new, ytrain)
 print(m.oob_score_)
 check_with = m.oob_score_
def create_add_model(input_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(input_dim, activation='elu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.4))
model.add(WeightNormalization(L.Dense(512, activation='selu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(y_non_train.shape[1], activation='sigmoid',
bias_initializer=tf.keras.initializers.Constant(-np.log(y_non_train.mean(axis=0 ).to_numpy())
))))
model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=1e-3), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model<create_dataframe>
non_scored_template = sample_submission.copy()
non_scored_template = pd.DataFrame(non_scored_template.pop('sig_id'))
for col in non_scored_target_cols:
non_scored_template[col] = 0 .<predict_on_test>
m = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1)
 m.fit(xtrain_new, ytrain)
 print_score(m,(xtrain_new, ytrain),(xvalid_new, yvalid))
if ADD_NON_TRAIN:
add_result, add_valid_preds, add_metrics = train_and_predict('ADD', create_add_model, X_train, y_non_train, X_test,
train_targets_nonscored[non_scored_target_cols].loc[train_trt_index,:], non_scored_template,
ADD_SEEDS, ADD_SPLITS, ADD_EPOCHS, ADD_BATCH_SIZE, shuffle_rows=False, do_show_metrics=False, show_each_metrics=False )<normalization>
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold, RandomizedSearchCV
if ADD_NON_TRAIN:
X_non_train = add_valid_preds.copy()
X_non_test = add_result.iloc[:,1:].copy()
cols = X_non_train.columns
X_non_train, X_non_test = rank_gauss(X_non_train, X_non_test, cols)
 X_non_train, X_non_test = standard_scaler(X_non_train, X_non_test, cols )<categorify>
def create_autoencoder(input_dim):
input_vector = L.Input(shape=(input_dim,))
encoded = L.Dense(1024, activation='elu' )(input_vector)
 encoded = L.Dense(2048, activation='elu' )(encoded)
 decoded = L.Dense(1024, activation='elu' )(encoded)
 decoded = L.Dense(input_dim, activation='elu' )(decoded)
 autoencoder = M.Model(input_vector, decoded)
 autoencoder.compile(optimizer=tf.optimizers.Adam(lr=0.0012, amsgrad=True), loss='mse')
 return autoencoder<categorify>
def learningRateAnnl(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True )
mu, sigma = 0, 0.1
seed_everything(128)
 dae_train = pd.concat([_X_train_dae, _X_test_dae])
 noise = np.random.normal(mu, sigma, [dae_train.shape[0], dae_train.shape[1]])
 noised_train = dae_train + noise
autoencoder = create_autoencoder(dae_train.shape[1])
 autoencoder.fit(noised_train, dae_train,
epochs=500,
batch_size=128,
callbacks=[
ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience = 3, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'dae.hdf5', monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 , EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 8, mode = 'min', baseline = None, restore_best_weights = True)],
shuffle=True,
validation_split=0.2)
 autoencoder.load_weights(f'dae.hdf5')
<categorify>
fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(xvalid3, yvalid3.copy() -1)],
'eval_names': ['valid'],
'callbacks': [lgb.reset_parameter(learning_rate=learningRateAnnl)],
'verbose': False,
'categorical_feature': 'auto'}
encoder = M.Model(autoencoder.input, autoencoder.layers[2].output)
 __X_train_dae = pd.DataFrame(encoder.predict(_X_train_dae))
__X_test_dae = pd.DataFrame(encoder.predict(_X_test_dae))<feature_engineering>
param_test ={'num_leaves': randint(12, 20),
'min_child_samples': randint(40, 120),
'subsample': uniform(loc=0.75, scale=0.20),
'colsample_bytree': uniform(loc=0.8, scale=0.15),
}
vh_dae = VarianceThreshold(0.2)
 vh_dae.fit_transform(pd.concat([__X_train_dae, __X_test_dae]))
dae_cols = __X_train_dae.columns[vh_dae.get_support() ]
print(len(dae_cols))
X_train_dae = __X_train_dae.loc[:, dae_cols]
X_test_dae = __X_test_dae.loc[:, dae_cols]<concatenate>
maxHPs = 400
classifier = lgb.LGBMClassifier(learning_rate=0.05, n_jobs=-1, n_estimators=500, objective='multiclass')
X_train_dae = pd.concat([X_train_dae, X_non_train.reset_index(drop=True)], axis=1)
 X_test_dae = pd.concat([X_test_dae, X_non_test.reset_index(drop=True)], axis=1 )<concatenate>
opt_parameters = {'colsample_bytree': 0.8755593602517565,
'min_child_samples': 51,
'num_leaves': 19,
'subsample': 0.9437154452377117}
_X_train_G_PCA_dae, _X_test_G_PCA_dae = yield_pca(_X_train, _X_test, 'Gd', GENES, 90, 128)
 _X_train_C_PCA_dae, _X_test_C_PCA_dae = yield_pca(_X_train, _X_test, 'Cd', CELLS, 27, 128)
 _X_train_G_PCA_Dense_dae, _X_test_G_PCA_Dense_dae = yield_pca(_X_train, _X_test, 'Gd', GENES, 8, 128)
 _X_train_C_PCA_Dense_dae, _X_test_C_PCA_Dense_dae = yield_pca(_X_train, _X_test, 'Cd', CELLS, 1, 128)
 X_train_dae = pd.concat([X_train_dae, _X_train_G_PCA_dae, _X_train_C_PCA_dae, _X_train_G_PCA_Dense, _X_train_C_PCA_Dense_dae], axis=1)
 X_test_dae = pd.concat([X_test_dae, _X_test_G_PCA_dae, _X_test_C_PCA_dae, _X_test_G_PCA_Dense, _X_test_C_PCA_Dense_dae], axis=1)
 X_train_dae, X_test_dae = rank_gauss(X_train_dae, X_test_dae, X_train_dae.columns, random_state=128)
 X_train_dae, X_test_dae = standard_scaler(X_train_dae, X_test_dae, X_train_dae.columns )<choose_model_class>
classifier = lgb.LGBMClassifier(**classifier.get_params())
 classifier.set_params(**opt_parameters)
 fit_params['verbose'] = 200
_ = classifier.fit(xtrain3,(ytrain3 ).copy() -1, **fit_params )
def create_simple_nn(input_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(input_dim, activation='selu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(512, activation='swish')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(206, activation='sigmoid', bias_initializer=output_bias)))
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=0.01), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model<predict_on_test>
kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 for trn_idx, tst_idx in kf.split(train.drop(['Target'], axis=1), train['Target']):
xtr, xval = train.drop(['Target'], axis=1 ).iloc[trn_idx], train.drop(['Target'], axis=1 ).iloc[tst_idx]
ytr, yval = train['Target'].iloc[trn_idx].copy() -1, train['Target'].iloc[tst_idx].copy() -1
classifier.fit(xtr, ytr, eval_set=[(xval, yval)],
early_stopping_rounds=300, verbose=200 )
if RUN_SNN:
snn_result, snn_valid_preds, snn_metrics = train_and_predict('SNN', create_simple_nn, X_train_dae, _y_train, X_test_dae,
train_targets_scored.iloc[:,1:], sample_submission,
SNN_SEEDS, SNN_SPLITS, SNN_EPOCHS, SNN_BATCH_SIZE, shuffle_rows=False )<concatenate>
preds = classifier.predict(xvalid )
_X_train = pd.concat([X_train, X_non_train], axis=1)
 _X_test = pd.concat([X_test, X_non_test], axis=1 )<drop_column>
classifier2 = lgb.LGBMClassifier(**classifier.get_params())
 kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 for trn_idx, tst_idx in kf.split(train2.drop(['Target'], axis=1), train2['Target']):
xtr, xval = train2.drop(['Target'], axis=1 ).iloc[trn_idx].copy() , train2.drop(['Target'], axis=1 ).iloc[tst_idx].copy()
ytr, yval = train2['Target'].iloc[trn_idx].copy() -1, train2['Target'].iloc[tst_idx].copy() -1
classifier2.fit(xtr, ytr, eval_set=[(xval, yval)],
early_stopping_rounds=300, verbose=200 )
sep_cols = BASE_COLS + GENES
_X_train_1 = _X_train[sep_cols]
_X_train_2 = _X_train.drop(sep_cols,axis=1)
 _X_test_1 = _X_test[sep_cols]
_X_test_2 = _X_test.drop(sep_cols,axis=1)
 _X_train = [_X_train_1, _X_train_2]
_X_test = [_X_test_1, _X_test_2]<choose_model_class>
from torch.utils.data import TensorDataset, DataLoader
from torch.autograd.variable import Variable
def create_res_model(n_features, n_features_2):
print(f'the input dim is {n_features}, {n_features_2}')
 input_1 = L.Input(shape =(n_features,), name = 'Input1')
 input_2 = L.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = M.Sequential([
L.BatchNormalization() ,
L.Dropout(0.3),
L.Dense(512, activation='elu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(256, activation='elu')
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = L.Concatenate()([input_2, input_3])
 head_2 = M.Sequential([
L.BatchNormalization() ,
L.Dropout(0.3),
L.Dense(n_features_2, activation='relu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(n_features_2, activation='elu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(256, activation='relu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(256, activation='selu')
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = L.Average()([input_3, input_4])
 head_3 = M.Sequential([
L.BatchNormalization() ,
L.Dropout(0.3),
L.Dense(256, activation='swish'),
L.BatchNormalization() ,
L.Dense(256, activation='selu'),
L.BatchNormalization() ,
L.Dense(206, activation='sigmoid')
 ],name='Head3')
 output = head_3(input_4_avg)
 model = M.Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer=tf.optimizers.Adam(lr=0.002),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model<predict_on_test>
Ttrain = TensorDataset(torch.DoubleTensor(np.array(xtrain.values, dtype="float32")) ,
torch.LongTensor(np.array(ytrain.values, dtype="float32")-1))
if RUN_NN:
nn_result, nn_valid_preds, nn_metrics = train_and_predict('RES', create_res_model, _X_train, y_train, _X_test,
train_targets_scored.iloc[:,1:], sample_submission,
NN_SEEDS, NN_SPLITS, NN_EPOCHS, NN_BATCH_SIZE, shuffle_rows=False )<concatenate>
trainLoader = DataLoader(Ttrain, batch_size = 20, shuffle=True )
if ADD_NON_TRAIN:
_X_train = []
_X_train.append(X_train.copy())
 _X_train.append(X_non_train.copy())
 _X_test = []
_X_test.append(X_test.copy())
 _X_test.append(X_non_test.copy())
 _X_train[1] = pd.concat([_X_train[1], _X_train[0][CELLS]], axis=1)
 _X_test[1] = pd.concat([_X_test[1], _X_test[0][CELLS]], axis=1)
 _X_train[0] = _X_train[0].drop(CELLS, axis=1)
 _X_test[0] = _X_test[0].drop(CELLS, axis=1 )<define_search_model>
class Net(nn.Module):
def __init__(self, n_cols):
super(Net, self ).__init__()
self.first = nn.Sequential(
 nn.BatchNorm1d(n_cols),
nn.Linear(n_cols, 10),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(10, 4),
nn.Softmax(dim=1)
)
 def forward(self, x):
return self.first(x)
 net = Net(len(xtrain.columns)).double()
def create_twohead_model(input_dim1, input_dim2):
print(f'the input dim is {input_dim1}, {input_dim2}')
 input_1 = L.Input(input_dim1)
 x1 = L.BatchNormalization()(input_1)
 x1 = L.Dropout(0.5 )(x1)
 o1 = L.Dense(input_dim1, activation='elu' )(x1)
 input_2 = L.Input(input_dim2)
 x2 = L.BatchNormalization()(input_2)
 x2 = L.Dropout(0.5 )(x2)
 o2 = L.Dense(input_dim2, activation='elu' )(x2)
 x = L.Concatenate()([o1, o2])
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.4 )(x)
 x = L.Dense(512, activation='selu' )(x)
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.5 )(x)
 output = L.Dense(206, activation='sigmoid' )(x)
 model = M.Model([input_1, input_2], output)
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=0.01), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model<predict_on_test>
loss = nn.CrossEntropyLoss()
metrics = [F1score]
opt = optim.Adam(net.parameters() , weight_decay=1e-3)
if RUN_NN2:
nn2_result, nn2_valid_preds, nn2_metrics = train_and_predict('NN2', create_twohead_model, _X_train, y_train, _X_test,
train_targets_scored.iloc[:,1:], sample_submission,
NN2_SEEDS, NN2_SPLITS, NN2_EPOCHS, NN2_BATCH_SIZE, shuffle_rows=False )<define_variables>
def fit(model, lr, xtr, ytr, xvl, yvl, train_dl, n_epochs, loss, opt, metrics, annln=False, mult_dec=True):

 if(annln): annl = lrAnnealing(lr, 40, 449, mult_dec)
 for epoch in range(n_epochs):
tl = iter(train_dl)
 length = len(train_dl)
 for t in range(length):
xt, yt = next(tl)
 y_pred = model(Variable(xt))
l = loss(y_pred, Variable(yt))
if(annln): annl(opt)
 opt.zero_grad()
l.backward()
opt.step()
val_score = get_f1score(model,
torch.DoubleTensor(np.array(xvl, dtype = "float32")) ,
torch.LongTensor(np.array(yvl, dtype = "float32")-1))
trn_score = get_f1score(model,
torch.DoubleTensor(np.array(xtr, dtype = "float32")) ,
torch.LongTensor(np.array(ytr, dtype = "float32")-1))
if(epoch+1)%5 == 0:
print("Epoch " + str(epoch)+ "::"
+ " trnF1score: " + str(trn_score)
 +", valF1score: " + str(val_score))
def get_f1score(model, x, y):

 pred = model(Variable(x ).contiguous())
 ypreds = np.argmax(pred.contiguous().data.numpy() , axis=1)
 yactuals = y.contiguous().numpy()
return F1score(yactuals, ypreds)
 def set_lr(opt, lr):

 for pg in opt.param_groups: pg['lr'] = lr
class lrAnnealing() :
def __init__(self, ini_lr, epochs, itr_per_epoch, mult_dec):

 self.epochs = epochs
self.ipe = itr_per_epoch
self.m_dec = mult_dec
self.ppw =(self.ipe * self.epochs)// 4
self.count = 0
self.lr = ini_lr
self.values = np.cos(np.linspace(np.arccos(self.lr), np.arccos(1e-10), self.ppw))
self.mult = 1
def __call__(self, opt):

 self.count += 1
set_lr(opt, self.values[self.count-1]*self.mult)
 if self.count == len(self.values):
self.count = 0
if(self.m_dec): self.mult /= 2
top_feats = [ 0, 1, 2, 4, 6, 9, 11, 12, 13, 15, 16,
17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29,
31, 33, 36, 37, 39, 40, 41, 42, 44, 45, 47,
48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 60,
61, 65, 67, 68, 69, 71, 72, 73, 75, 76, 78,
79, 80, 81, 82, 83, 85, 86, 87, 88, 89, 91,
92, 93, 94, 96, 98, 99, 100, 101, 102, 103, 104,
107, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118,
119, 120, 122, 123, 124, 125, 126, 128, 129, 130, 132,
133, 134, 135, 136, 137, 140, 141, 142, 143, 144, 146,
148, 149, 150, 151, 152, 153, 154, 155, 158, 159, 161,
162, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173,
175, 178, 181, 183, 184, 185, 186, 187, 189, 190, 191,
193, 197, 198, 200, 201, 202, 203, 204, 205, 206, 209,
210, 211, 213, 214, 216, 218, 219, 220, 221, 223, 225,
226, 227, 229, 230, 231, 232, 233, 236, 237, 239, 241,
242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 254,
255, 256, 257, 258, 261, 262, 263, 265, 266, 268, 270,
271, 272, 273, 275, 276, 277, 279, 280, 281, 283, 284,
288, 289, 291, 294, 297, 298, 299, 300, 302, 303, 304,
305, 306, 309, 310, 311, 312, 314, 315, 316, 317, 319,
321, 322, 323, 324, 325, 326, 329, 330, 332, 336, 337,
339, 340, 341, 342, 343, 344, 347, 348, 349, 351, 354,
355, 356, 357, 358, 359, 360, 361, 362, 365, 367, 369,
370, 371, 373, 374, 375, 376, 378, 379, 380, 383, 384,
385, 388, 390, 392, 395, 397, 400, 401, 403, 404, 406,
407, 408, 410, 414, 415, 416, 418, 419, 420, 422, 423,
424, 425, 426, 427, 428, 430, 431, 433, 434, 436, 438,
439, 441, 443, 446, 447, 448, 450, 451, 452, 454, 455,
457, 461, 462, 463, 464, 466, 468, 469, 470, 473, 474,
475, 477, 478, 479, 481, 482, 483, 484, 487, 488, 489,
492, 495, 497, 501, 502, 503, 505, 506, 508, 509, 510,
511, 512, 513, 514, 516, 517, 519, 522, 523, 524, 525,
527, 528, 529, 531, 538, 543, 546, 548, 549, 550, 552,
553, 554, 555, 558, 559, 560, 561, 562, 563, 564, 565,
567, 568, 569, 572, 573, 574, 575, 576, 579, 580, 583,
585, 587, 588, 591, 592, 595, 596, 597, 599, 600, 603,
610, 611, 612, 613, 614, 617, 619, 620, 621, 623, 624,
625, 627, 628, 629, 632, 634, 635, 636, 637, 638, 639,
640, 642, 643, 644, 645, 646, 647, 649, 650, 651, 654,
656, 659, 660, 661, 662, 663, 665, 669, 671, 672, 673,
674, 676, 677, 678, 680, 681, 682, 685, 686, 687, 689,
692, 694, 695, 696, 697, 698, 699, 701, 702, 704, 705,
706, 708, 709, 710, 714, 716, 717, 718, 722, 724, 725,
729, 732, 733, 734, 735, 736, 737, 738, 742, 743, 748,
749, 751, 753, 754, 755, 756, 757, 759, 761, 763, 764,
765, 767, 768, 771, 772, 773, 774, 775, 776, 778, 779,
781, 782, 784, 786, 788, 791, 792, 793, 794, 800, 801,
802, 805, 807, 814, 815, 816, 820, 821, 822, 823, 824,
825, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837,
838, 839, 840, 841, 842, 843, 844, 846, 847, 848, 849,
850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860,
861, 863, 865, 866, 867, 868, 869, 870, 871, 872, 873,
876, 877, 878, 879, 880, 881, 882, 883, 884, 886, 887,
889, 890, 892, 893, 894, 895, 896, 897, 898, 899, 901,
902, 904, 905, 906, 909, 910, 911, 912, 913, 914, 915,
916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926,
927, 928, 929, 930, 931, 932, 933, 934, 938, 941, 942,
944, 948, 949, 950, 951, 954, 955, 956, 957, 958, 959,
960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970,
971, 972, 974, 976, 980, 981, 985, 987, 988, 989, 990,
991, 993, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003,
1004, 1005, 1006, 1010, 1011, 1012, 1013, 1014, 1016, 1017, 1018,
1019, 1020, 1021, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1031,
1032, 1034, 1035, 1037, 1038, 1040, 1041, 1042, 1046, 1047, 1048,
1049, 1050, 1051, 1053, 1055, 1056, 1057, 1058, 1059, 1060, 1062,
1064, 1065, 1066, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1076,
1078, 1079, 1080, 1082, 1083, 1084, 1085, 1086, 1087, 1089, 1092,
1093, 1094, 1095, 1096, 1097, 1099, 1100, 1101, 1102, 1103, 1104,
1105, 1106, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116,
1117, 1118, 1119, 1120, 1121, 1122, 1125, 1126, 1127, 1128, 1131,
1132, 1134, 1136, 1137, 1138, 1140, 1141]<concatenate>
warnings.filterwarnings("ignore", category=UndefinedMetricWarning )
X_train = pd.concat([X_train_nl, X_non_train], axis=1)
 X_test = pd.concat([X_test_nl, X_non_test], axis=1 )<split>
%time fit(net, 1e-2, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )
X_train_rnn = X_train.iloc[:, top_feats]
X_test_rnn = X_test.iloc[:, top_feats]<choose_model_class>
%time fit(net, 1e-3, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )
def create_rnn_model(input_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.Reshape(( 1, input_dim)))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.3))
model.add(L.GRU(1024, dropout = 0.5, recurrent_dropout=0.3, return_sequences = True, activation='elu'))
model.add(L.GRU(1024, dropout = 0.5, recurrent_dropout=0.5, activation='selu'))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(206, activation='sigmoid', bias_initializer=output_bias)))
 model.compile(optimizer = tfa.optimizers.Lookahead(tf.optimizers.Adam(0.0015), sync_period = 2),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model<predict_on_test>
%time fit(net, 1e-4, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )
if RUN_RNN:
rnn_result, rnn_valid_preds, rnn_metrics = train_and_predict('RNN', create_rnn_model, X_train_rnn, y_train, X_test_rnn,
train_targets_scored.iloc[:,1:], sample_submission,
RNN_SEEDS, RNN_SPLITS, RNN_EPOCHS, RNN_BATCH_SIZE )<import_modules>
%time fit(net, 1e-5, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except KeyError:
raise KeyError("Your zip file is missing at least one component")
 self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss = self.log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 y_smo = targets.float() *(1 - LABEL_SMOOTHING)+ 0.5 * LABEL_SMOOTHING
loss = self.loss_fn(output, y_smo)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def log_loss_multi(self, y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = self.log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
def log_loss_score(self, actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss = self.log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 y_smo = targets.float() *(1 - LABEL_SMOOTHING)+ 0.5 * LABEL_SMOOTHING
loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res<predict_on_test>
%time fit(net, 1e-6, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )
def train_and_predict(name, _X_i_train, _y_i_train, _X_i_test, seeds, splits, epochs, batch_size, shuffle_rows=False, pick_col_size=800):
st = time.time()
val_result = train_targets_scored.iloc[:,1:].copy()
val_result.loc[:, :] = 0
sub_result = sample_submission.copy()
sub_result.loc[:, 1:] = 0
all_cols = _X_i_train.columns.values
for h, seed in enumerate(seeds):
seed_everything(seed)
 for i,(train_idx, valid_idx)in enumerate(MultilabelStratifiedKFold(n_splits=splits, random_state=seed, shuffle=True)
.split(_y_i_train, _y_i_train)) :
print(f'Fold {i+1}')
 if shuffle_rows:
seed_everything(seed + i)
 target_cols = np.random.choice(all_cols, size=pick_col_size)
 seed_everything(seed)
 else:
target_cols = all_cols
_X_train = torch.as_tensor(_X_i_train.loc[:,target_cols].values[train_idx])
 _y_train = torch.as_tensor(_y_i_train.values[train_idx])
 _X_valid = torch.as_tensor(_X_i_train.loc[:,target_cols].values[valid_idx])
 _y_valid = torch.as_tensor(_y_i_train.values[valid_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5), mask_type='entmax',
device_name="cuda" if torch.cuda.is_available() else "cpu",
scheduler_params=dict(milestones=[100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 model.fit(X_train=_X_train, y_train=_y_train, X_valid=_X_valid, y_valid=_y_valid, max_epochs=epochs, patience=20, batch_size=1024, virtual_batch_size=batch_size,
num_workers=0, drop_last=False, loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 model.load_best_model()
preds = model.predict(_X_valid)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
val_result.iloc[y_train.iloc[valid_idx,:].index.values, :] += preds
preds = model.predict(torch.as_tensor(_X_i_test.loc[test_trt_index, target_cols].values))
preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
sub_result.loc[test_trt_index, sub_result.columns[1:]] += preds
print('')
 tmp_result = val_result.copy()
tmp_result.iloc[:,1:] = val_result.iloc[:,1:] /(h + 1)
 print(f' ---- seed:{seed}, ensemble:{h + 1}')
 _ = show_metrics(tmp_result, show_each=False)
 val_result.iloc[:,1:] = val_result.iloc[:,1:] / len(seeds)
 metrics = show_metrics(val_result)
 sub_result.iloc[:, 1:] = sub_result.iloc[:, 1:] /(len(seeds)* splits)
 print(f' elapsed: {time.time() - st}')
 return sub_result, val_result, metrics<predict_on_test>
if RUN_TABNET:
tab_result, tab_valid_preds, tab_metrics = train_and_predict('TAB', X_train, y_train, X_test,
TAB_SEEDS, TAB_SPLITS, TAB_EPOCHS, TAB_BATCH_SIZE )<save_to_csv>
if not RE_RUN_NN:
ens_result.to_csv('submission.csv', index=False )<predict_on_test>
def re_train_and_predict(name, model_func, _X_i_train, _y_i_train, _X_i_test, seeds, splits, epochs, batch_size, shuffle_rows=False, pick_col_size=800):
st = time.time()
val_result = train_targets_scored.loc[:,_y_i_train.columns].copy()
val_result.loc[:, :] = 0
sub_result = sample_submission.loc[:, _y_i_train.columns].copy()
sub_result.loc[:, :] = 0
all_cols = _X_i_train.columns.values
for h, seed in enumerate(seeds):
seed_everything(seed)
 for i,(train_idx, valid_idx)in enumerate(MultilabelStratifiedKFold(n_splits=splits, random_state=seed, shuffle=True)
.split(_y_i_train, _y_i_train)) :
print(f'Fold {i+1}')
 if shuffle_rows:
seed_everything(seed + i)
 target_cols = np.random.choice(all_cols, size=pick_col_size)
 seed_everything(seed)
 else:
target_cols = all_cols
_X_train = _X_i_train.loc[:,target_cols].values[train_idx]
_y_train = _y_i_train.values[train_idx]
_X_valid = _X_i_train.loc[:,target_cols].values[valid_idx]
_y_valid = _y_i_train.values[valid_idx]
model = model_func(len(target_cols), len(_y_i_train.columns))
model.fit(_X_train, _y_train,
validation_data=(_X_valid, _y_valid),
epochs=epochs, batch_size=batch_size,
callbacks=[
ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=3, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'{name}_{seed}_{i}.hdf5', monitor = 'val_logloss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 , EarlyStopping(monitor = 'val_logloss', min_delta = 1e-4, patience = 5, mode = 'min', baseline = None, restore_best_weights = True)
 ], verbose=2)
 model.load_weights(f'{name}_{seed}_{i}.hdf5')
 val_result.iloc[y_train.iloc[valid_idx,:].index.values, :] += model.predict(_X_valid)
 sub_result.loc[test_trt_index, :] += model.predict(_X_i_test.loc[test_trt_index, target_cols])
 print('')
 tmp_result = val_result.copy()
tmp_result.iloc[:,1:] = val_result.iloc[:,1:] /(h + 1)
 print(f' ---- seed:{seed}, ensemble:{h + 1}')
 _ = show_metrics(tmp_result, show_each=False)
 val_result.iloc[:,1:] = val_result.iloc[:,1:] / len(seeds)
 metrics = show_metrics(val_result)
 sub_result.iloc[:, :] = sub_result.iloc[:, :] /(len(seeds)* splits)
 print(f' elapsed: {time.time() - st}')
 return sub_result, val_result, metrics<concatenate>
X_stacking_train = pd.concat([nn_valid_preds, rnn_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = pd.concat([X_stacking_train, tab_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = pd.concat([X_stacking_train, nn2_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = pd.concat([X_stacking_train, snn_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = X_stacking_train[train_trt_index]
X_stacking_test = pd.concat([nn_result.iloc[:, 1:], rnn_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = pd.concat([X_stacking_test, tab_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = pd.concat([X_stacking_test, nn2_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = pd.concat([X_stacking_test, snn_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = X_stacking_test<choose_model_class>
m = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1)
 m.fit(xtrain, ytrain)
 print_score(m,(xtrain, ytrain),(xvalid, yvalid))
to_pred = test[xtrain.columns]
npArray = np.stack([lb3.inverse_transform(test['Id'].values), m.predict(to_pred)], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission1.csv", index=False )
def create_stacking_model(input_dim, output_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.3))
model.add(WeightNormalization(L.Dense(input_dim, activation='elu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(input_dim, activation='swish')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(output_dim, activation='sigmoid', bias_initializer=output_bias)))
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=1e-3), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model<predict_on_test>
m2 = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1)
 m2.fit(xtrain2, ytrain2)
 to_pred = test[xtrain2.columns]
npArray = np.stack([lb3.inverse_transform(test['Id'].values), m2.predict(to_pred)], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission2.csv", index=False )
if RUN_STACKING:
st_re_result, st_re_valid_preds, st_re_metrics = re_train_and_predict('ST_NN', create_stacking_model,
X_stacking_train, y_train, X_stacking_test,
STK_SEEDS, STK_SPLITS, STK_EPOCHS, STK_BATCH_SIZE )<load_pretrained>
m3 = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1, class_weight='balanced')
 m3.fit(xtrain3, ytrain3)
 to_pred = test[xtrain3.columns]
npArray = np.stack([lb3.inverse_transform(test['Id'].values), m3.predict(to_pred)], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission3.csv", index=False )
with open('nn_valid_preds.pkl', 'wb')as p:
pickle.dump(nn_valid_preds , p)
 with open('nn2_valid_preds.pkl', 'wb')as p:
pickle.dump(nn2_valid_preds , p)
 with open('rnn_valid_preds.pkl', 'wb')as p:
pickle.dump(rnn_valid_preds , p)
 with open('tab_valid_preds.pkl', 'wb')as p:
pickle.dump(tab_valid_preds , p)
 with open('st_re_valid_preds.pkl', 'wb')as p:
pickle.dump(st_re_valid_preds , p)
 with open('snn_valid_preds.pkl', 'wb')as p:
pickle.dump(snn_valid_preds , p )<compute_train_metric>
pred = net(Variable(torch.DoubleTensor(np.array(test.drop(['Id', 'idhogar'], axis=1 ).values, dtype="float32")) ).contiguous())
 ypreds = pred.contiguous().data.numpy().argmax(1)+ 1
npArray = np.stack([lb3.inverse_transform(test['Id'].values),ypreds], axis=-1); npArray[0]
res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission4.csv", index=False )
ratio = [0.05, 0.05, 0.1, 0.25, 0.35, 0.2]
ens_valid_preds = nn_valid_preds * ratio[0] + nn2_valid_preds * ratio[1] + rnn_valid_preds * ratio[2] + tab_valid_preds * ratio[3] + \
st_re_valid_preds * ratio[4] + snn_valid_preds * ratio[5]
_ = show_metrics(ens_valid_preds, show_each=False )<save_to_csv>
ypreds = classifier.predict(test[xtrain.columns])+ 1
npArray = np.stack([lb3.inverse_transform(test['Id'].values),ypreds], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission5.csv", index=False )
<install_modules><EOS> 
ypreds = classifier2.predict(test[xtrain2.columns])+ 1
npArray = np.stack([lb3.inverse_transform(test['Id'].values),ypreds], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission6.csv", index=False )