Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<define_variables>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<init_hyperparams>
pd.options.display.max_columns = None
class Config:
def __init__(self):
self.cat = []
self.ordinal = ['cp_time', 'cp_dose']
self.n_features = 838
self.n_classes = 206
self.n_classes_nonscored = 402
self.batch_size = 1024
self.num_epoch = 120
self.model_path = f'{KAGGLE_PATH}/working/'
self.drop_features = ['sig_id', 'cp_type', 'fold']
self.n_folds = 7
self.weight_decay = 1e-5
self.early_stopping_steps = 20
self.learning_rate = 4e-3
self.learning_rate_nonscored = 4e-3
self.hidden_size = 1024
self.seeds = [i for i in range(5)]
def seed_everything(seed_value=228):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
config = Config()
seed_everything()<data_type_conversions>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
class MoaDataset(Dataset):
def __init__(self, features, targets=None, transforms=None):
self.features = torch.tensor(features.drop(columns=config.drop_features, errors='ignore' ).to_numpy().astype(np.float32)).squeeze()
if targets is not None:
self.targets = torch.tensor(targets.drop(columns=config.drop_features, errors='ignore' ).to_numpy().astype(np.float32)).squeeze()
else:
self.targets = None
self.ixs = features.sig_id.unique()
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
ix = self.ixs[index]
if self.targets is None:
return {
'id': ix,
'features': self.features[index],
}
else:
return {
'id': ix,
'features': self.features[index],
'targets': self.targets[index],
}
def random_split(self, val_size):
num_val = int(val_size * len(self))
num_train = len(self)- num_val
return random_split(self, [num_train, num_val])
 def get_targets(self):
return self.targets
def get_features(self):
return self.features<normalization>
mapping = {"yes": 1, "no": 0}
for df in [train, test]:
df['dependency'] = df['dependency'].replace(mapping ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(mapping ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(mapping ).astype(np.float64)
 train[['dependency', 'edjefa', 'edjefe']].describe()
class Layer(nn.Module):
def __init__(self, input_size, output_size, dropout, activation_f=F.leaky_relu):
super().__init__()
self.batch_norm = nn.BatchNorm1d(input_size)
 self.dropout = nn.Dropout(dropout)
 self.dense = nn.utils.weight_norm(nn.Linear(input_size, output_size))
self.activation_f = activation_f
def forward(self, x):
x = self.batch_norm(x)
 x = self.dropout(x)
 x = self.activation_f(self.dense(x))
return x
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=config.hidden_size):
super().__init__()
self.input_layer = Layer(num_features, hidden_size, 0.3)
 self.hidden_layer = Layer(hidden_size, hidden_size, 0.3)
 self.classifier = Layer(hidden_size, num_targets, 0.3, activation_f=torch.sigmoid)
 def forward(self, x):
x = self.input_layer(x)
 x = self.hidden_layer(x)
 x = self.classifier(x)
 return x<categorify>
test['Target'] = np.nan
data = train.append(test, ignore_index = True )
class basic_encoder:
@staticmethod
def merge_samples(samples):
for i in range(len(samples)) :
samples[i]['sample'] = i
mega_sample = pd.concat(samples)
 return mega_sample
@staticmethod
def split_samples(samples, mega_sample):
for i in range(len(samples)) :
samples[i].drop(columns='sample', inplace=True)
 samples[i] = mega_sample[mega_sample['sample'] == i].drop(columns='sample' ).reset_index(drop=True)
 return samples
class encode_categorical(basic_encoder):
def __init__(self, cat_features=config.cat):
self.cat_features = cat_features
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 mega_sample = pd.get_dummies(mega_sample, columns=self.cat_features)
 return self.split_samples(samples, mega_sample)
 class encode_ordinal(basic_encoder):
def __init__(self, ordinal_features=config.ordinal):
self.ordinal_features = ordinal_features
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 for of in self.ordinal_features:
for i, u in enumerate(mega_sample[of].unique()):
mega_sample.loc[mega_sample[of] == u, of] = i
return self.split_samples(samples, mega_sample)
 class encode_PCA(basic_encoder):
def __init__(self, columns, n_pca, prefix='pca_g-', ignore_cols=False):
self.columns = columns
self.n_pca = n_pca
self.prefix = prefix
self.ignore_cols = ignore_cols
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 if self.ignore_cols:
self.columns = set(self.columns ).intersection(mega_sample.columns)
 pca = PCA(n_components=self.n_pca, random_state=42, svd_solver='full')
 pca_sample = pd.DataFrame(pca.fit_transform(mega_sample[self.columns]))
pca_sample.columns = [self.prefix + str(i)for i in range(1, pca_sample.shape[1] + 1)]
mega_sample = pd.concat([mega_sample.reset_index(drop=True), pca_sample], axis=1)
 return self.split_samples(samples, mega_sample)
 class normalize(basic_encoder):
def __init__(self,):
pass
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 cols = [c for c in mega_sample.columns if c not in config.drop_features + ['sample']]
mega_sample.loc[:, cols] =(mega_sample.loc[:, cols] - mega_sample.loc[:, cols].mean())/ mega_sample.loc[:, cols].std()
return self.split_samples(samples, mega_sample)
 class drop_low_varience(basic_encoder):
def __init__(self, threshold=0.7):
self.threshold=threshold
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 cols_to_drop = mega_sample.var() [mega_sample.var() <= self.threshold].index.to_list()
cols_to_drop.remove('sample')
 mega_sample.drop(columns=cols_to_drop, inplace=True)
 return self.split_samples(samples, mega_sample)
 class rank_gauss(basic_encoder):
def __init__(self, n_quantiles=100):
self.n_quantiles=n_quantiles
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 transformer = QuantileTransformer(n_quantiles=self.n_quantiles, random_state=0, output_distribution="normal")
 cols = [c for c in mega_sample.columns if c not in config.drop_features + ['sample']]
mega_sample.loc[:, cols] = transformer.fit_transform(mega_sample[cols])
 return self.split_samples(samples, mega_sample)
 class rank_gauss2(basic_encoder):
def __init__(self):
pass
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 transformer = GaussRankScaler()
cols = [c for c in mega_sample.columns if c not in config.drop_features + ['sample']]
mega_sample.loc[:, cols] = transformer.fit_transform(mega_sample[cols])
 return self.split_samples(samples, mega_sample)
 class add_statistics(basic_encoder):
def __init__(self, stats, columns, prefix, ignore_cols=False):
self.stats = stats
self.columns = columns
self.prefix = prefix
self.ignore_cols = ignore_cols
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 if self.ignore_cols:
self.columns = set(self.columns ).intersection(mega_sample.columns)
 for stat in self.stats:
mega_sample[self.prefix + stat] = getattr(mega_sample[self.columns], stat )(axis=1)
 return self.split_samples(samples, mega_sample)
 def preprocess(samples, transforms):
for transform in transforms:
samples = transform(samples)
 return samples 
<categorify>
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
train = train_features[train_features.cp_type == 'trt_cp'].reset_index(drop=True)
 test = test_features[test_features.cp_type == 'trt_cp'].reset_index(drop=True)
 train_targets_scored_drop = train_targets_scored[train_features.cp_type == 'trt_cp'].reset_index(drop=True)
 train_targets_nonscored_drop = train_targets_nonscored[train_features.cp_type == 'trt_cp'].reset_index(drop=True)
 train, test = preprocess([train, test], [drop_low_varience() ,
encode_categorical() ,
encode_ordinal() ,
add_statistics(['sum', 'mean', 'std', 'kurt', 'skew'], GENES, 'g_', ignore_cols=True),
add_statistics(['sum', 'mean', 'std', 'kurt', 'skew'], CELLS, 'c_', ignore_cols=True),
drop_low_varience() ,
rank_gauss2() ])
<normalization>
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]
class SmoothCrossEntropyLoss(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0, logits=False):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
self.logits = logits
@staticmethod
def _smooth(targets, n_classes, smoothing=0.0):
assert 0 <= smoothing <= 1
with torch.no_grad() :
targets = targets *(1 - smoothing)+ torch.ones_like(targets ).to(device)* smoothing / n_classes
return targets
def forward(self, inputs, targets):
targets = SmoothCrossEntropyLoss()._smooth(targets, inputs.shape[1], self.smoothing)
 if self.weight is not None:
inputs = inputs * self.weight.unsqueeze(0)
 if self.logits:
loss = F.binary_cross_entropy_with_logits(inputs, targets)
 else:
loss = F.binary_cross_entropy(inputs, targets)
 return loss<train_model>
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))
def train_val_fn(model, optimizer, scheduler, loss_fn, metric, dataloader, device, dataset_sizes, phase='train', epoch=0, cur_val_los=None):
gc.collect()
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
epoch_metric = 0
preds = []
for batch in dataloader:
x = batch['features'].to(device)
 targets = batch['targets'].to(device)
 if phase == 'train':
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
pred = model(x)
 if not pred.isnan().any() :
loss = loss_fn(pred, targets)
 else:
model.zero_grad()
continue
if phase == 'train':
loss.backward()
optimizer.step()
else:
preds.append(pred)
 with torch.no_grad() :
epoch_metric += metric(pred, targets ).item() * batch['targets'].size(0)
 running_loss += loss.item() * batch['targets'].size(0)
 if phase == 'val':
try:
preds = np.concatenate([p.cpu().numpy() for p in preds])
 except:
return float('inf'), None, float('inf')
 return running_loss / dataset_sizes[phase], preds, epoch_metric / dataset_sizes[phase]
@torch.no_grad()
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for batch in dataloader:
x = batch['features']
x = x.to(device)
 pred = model(x)
 preds.append(pred)
 preds = np.concatenate([p.cpu().numpy() for p in preds])
 return preds<prepare_x_and_y>
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(households_no_head_equal == False)) )
def train_fold(features, targets, test, fold, verbose=10, from_pretrain=False):
t_ix = features[features['fold'] != fold].index
v_ix = features[features['fold'] == fold].index
X = {
'train': features.loc[t_ix],
'val': features.loc[v_ix]
}
y = {
'train': targets.loc[t_ix],
'val': targets.loc[v_ix]
}
if from_pretrain:
model = Model(config.n_features, config.n_classes_nonscored)
 model.load_state_dict(torch.load(config.model_path + f'{fold}_model.pt'))
model.classifier = Layer(config.hidden_size, config.n_classes, 0.25, torch.sigmoid)
 model = model.to(device)
 else:
model = Model(config.n_features, config.n_classes ).to(device)
 datasets = {sample: MoaDataset(X[sample], y[sample])
 for sample in ['train', 'val']}
dataloaders = {
sample: DataLoader(
 datasets[sample],
batch_size=config.batch_size,
shuffle=(sample == 'train'),
num_workers=0 
)for sample in ['train', 'val']}
dataset_sizes = {sample: len(datasets[sample])for sample in ['train', 'val']}
loss_fn = SmoothCrossEntropyLoss(smoothing=0.001)
 metric = nn.BCELoss()
learning_rate = config.learning_rate
num_train_steps = int(dataset_sizes['train'] / config.batch_size)* config.num_epoch
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=config.weight_decay)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, factor=0.5, patience=5)
 best_loss = float('inf')
 epoch_loss = float('inf')
 early_step = 0
oos = np.zeros_like(targets.drop(columns=config.drop_features, errors='ignore'), dtype=np.float)
 best_epoch = config.num_epoch
print(f' FOLD {fold} MODEL TRAINING '.ljust(99, '=' ).rjust(190, '='))
stop = False
for epoch in range(config.num_epoch):
for phase in ['train', 'val']:
epoch_loss, preds, epoch_metric = train_val_fn(model, optimizer, scheduler, loss_fn, metric, dataloaders[phase], device, dataset_sizes, phase, epoch, cur_val_los=epoch_loss)
 if phase == 'val':
scheduler.step(epoch_metric)
 if phase == 'val' and epoch_metric < best_loss:
best_loss = epoch_metric
best_model_wts = copy.deepcopy(model.state_dict())
 oos[v_ix] = preds
best_epoch = epoch
early_step = 0
elif phase == 'val':
if preds is None:
model.load_state_dict(best_model_wts)
 early_step += 1
if early_step >= config.early_stopping_steps:
stop = True
if epoch % verbose == 0:
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 if stop:
break
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 print(f"Best epoch {best_epoch} validation loss: {best_loss}")
 torch.save(best_model_wts, config.model_path + f'{fold}_model.pt')
 model.load_state_dict(best_model_wts)
 dataset_test = MoaDataset(test)
 dataloader_test = DataLoader(
 dataset_test,
batch_size=config.batch_size,
shuffle=False,
num_workers=0)
 test_pred = inference_fn(model, dataloader_test, device)
 return oos, test_pred
def pretrain_fold(features, targets, fold, verbose=10):
t_ix = features[features['fold'] != fold].index
v_ix = features[features['fold'] == fold].index
X = {
'train': features.loc[t_ix],
'val': features.loc[v_ix]
}
y = {
'train': targets.loc[t_ix],
'val': targets.loc[v_ix]
}
model = Model(config.n_features, config.n_classes_nonscored ).to(device)
 datasets = {sample: MoaDataset(X[sample], y[sample])
 for sample in ['train', 'val']}
dataloaders = {
sample: DataLoader(
 datasets[sample],
batch_size=config.batch_size,
shuffle=(sample == 'train'),
num_workers=0 
)for sample in ['train', 'val']}
dataset_sizes = {sample: len(datasets[sample])for sample in ['train', 'val']}
loss_fn = SmoothCrossEntropyLoss(smoothing=0.1)
 metric = nn.BCELoss()
learning_rate = config.learning_rate_nonscored
num_train_steps = int(dataset_sizes['train'] / config.batch_size)* config.num_epoch
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=config.weight_decay)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, factor=0.5, patience=5)
 best_loss = float('inf')
 epoch_loss = float('inf')
 early_step = 0
best_epoch = config.num_epoch
print(f' FOLD {fold} MODEL PRETRAINING '.ljust(99, '=' ).rjust(190, '='))
stop = False
for epoch in range(config.num_epoch):
for phase in ['train', 'val']:
epoch_loss, preds, epoch_metric = train_val_fn(model, optimizer, scheduler, loss_fn, metric, dataloaders[phase], device, dataset_sizes, phase, epoch, cur_val_los=epoch_loss)
 if phase == 'val':
scheduler.step(epoch_metric)
 if phase == 'val' and epoch_metric < best_loss:
best_loss = epoch_metric
best_model_wts = copy.deepcopy(model.state_dict())
 best_epoch = epoch
early_step = 0
elif phase == 'val':
if preds is None:
model.load_state_dict(best_model_wts)
 early_step += 1
if early_step >= config.early_stopping_steps:
stop = True
if epoch % verbose == 0:
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 if stop:
break
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 print(f"Best epoch {best_epoch} validation loss: {best_loss}")
 torch.save(best_model_wts, config.model_path + f'{fold}_model.pt' )<categorify>
for household in not_equal.index:
true_target = int(train[(train['idhogar'] == household)&(train['parentesco1'] == 1.0)]['Target'])
 train.loc[train['idhogar'] == household, 'Target'] = true_target
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
def run_k_fold(NFOLDS, verbose=10, from_pretrain=False):
oos = np.zeros_like(train_targets_scored_drop.drop(columns=config.drop_features, errors='ignore'), dtype=np.float)
 predictions = np.zeros(( test.shape[0], config.n_classes), dtype=np.float)
 for fold in range(NFOLDS):
oos_, pred_ = train_fold(train, train_targets_scored_drop, test, fold, verbose, from_pretrain)
 predictions += pred_ / NFOLDS
oos += oos_
return oos, predictions
def run_k_fold_pretrain(NFOLDS, verbose=10):
for fold in range(NFOLDS):
pretrain_fold(train, train_targets_nonscored_drop, fold, verbose)
 def run_n_seeds(train_df, targets, targets_nonscored=None, SEEDS=config.seeds, verbose=10, pretrain=False, eval=False):
assert pretrain == False or pretrain == True and targets_nonscored is not None
oos = []
test_pred = []
for seed in SEEDS:
print(f' SEED {seed} '.ljust(90, '*' ).rjust(190, '*'))
seed_everything(seed)
 kf = MultilabelStratifiedKFold(config.n_folds, shuffle=True, random_state=seed)
 targets_to_strat = targets.merge(pd.get_dummies(train_drug, columns=['drug_id']), on='sig_id', how='left')
 if pretrain:
for f,(t_idx, v_idx)in enumerate(kf.split(X=train, y=targets_to_strat.drop(columns=config.drop_features, errors='ignore'))):
train.loc[v_idx, 'fold'] = f
train['fold'] = train['fold'].astype(int)
 run_k_fold_pretrain(config.n_folds)
 oos_seed, test_pred_seed = run_k_fold(config.n_folds, verbose=verbose, from_pretrain=True)
 else:
for f,(t_idx, v_idx)in enumerate(kf.split(X=train, y=targets_to_strat.drop(columns=config.drop_features, errors='ignore'))):
train.loc[v_idx, 'fold'] = f
train['fold'] = train['fold'].astype(int)
 oos_seed, test_pred_seed = run_k_fold(config.n_folds, verbose=verbose, from_pretrain=False)
 oos.append(oos_seed)
 test_pred.append(test_pred_seed)
 if eval:
print('*'*20)
 print(f'LOSS SEED {seed}:', nn.BCELoss()(torch.tensor(oos_seed, dtype=torch.float), torch.tensor(targets.drop(columns=config.drop_features, errors='ignore' ).to_numpy() , dtype=torch.float)))
 print('*'*20)
 return np.mean(oos, 0), np.mean(test_pred, 0 )<feature_engineering>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 ).drop('Target' )
os.environ['CUDA_LAUNCH_BLOCKING'] = "1"
config.n_features = train.drop(columns=config.drop_features, errors='ignore' ).shape[1]
t0 = time()
date_time = datetime.now().strftime("%Y%m%d-%H%M")
 oos, test_pred = run_n_seeds(train, train_targets_scored_drop, train_targets_nonscored_drop, pretrain=True, eval=True)
 print(f'Done! Time {timedelta(seconds=time() - t0)}' )<merge>
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )
oos_df = pd.DataFrame(oos)
 oos_df.insert(0, 'sig_id', train.sig_id)
 oos_df.columns = train_targets_scored.columns
oos_df = train_targets_scored[['sig_id']].merge(oos_df, on='sig_id', how='left' ).fillna(0 )<data_type_conversions>
data['v18q1'] = data['v18q1'].fillna(0 )
print('loss without ctl: ', nn.BCELoss()(torch.tensor(oos, dtype=torch.float), torch.tensor(train_targets_scored_drop.drop(columns=config.drop_features,errors='ignore' ).to_numpy() , dtype=torch.float)).item())
 print('loss with ctl: ', nn.BCELoss()(torch.tensor(oos_df.drop(columns='sig_id' ).to_numpy() , dtype=torch.float), torch.tensor(train_targets_scored.drop(columns='sig_id' ).to_numpy() , dtype=torch.float)).item() )<merge>
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()
test_pred_df = pd.DataFrame(test_pred)
 test_pred_df.insert(0, 'sig_id', test.sig_id)
 test_pred_df.columns = sample_submission.columns
test_pred_df = sample_submission[['sig_id']].merge(test_pred_df, on='sig_id', how='left' ).fillna(0 )<save_to_csv>
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
test_pred_df.to_csv('submission.csv', index=False )<import_modules>
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5
import numpy as np
import pandas as pd
import os<install_modules>
id_ = ['Id', 'idhogar', 'Target']
!pip install pytorch-tabnet<install_modules>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
!pip install.. /input/iterative-stratification/iterative_stratification-0.1.6-py3-none-any.whl 
<feature_engineering>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
sys.path.append('.. /input/iterativestratification')
 os.environ["CUDA_LAUNCH_BLOCKING"] = "1" 
<set_options>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<load_from_csv>
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<create_dataframe>
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
n_comp = 610
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 55
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True)
var_thresh = VarianceThreshold(0.82)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<data_type_conversions>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 heads = heads.drop(columns = ['eviv1', 'eviv2', 'eviv3'] )
def get_ratio_labels(df):
columns = list(df.columns)
 columns.pop(0)
 ratios = []
toremove = []
for c in columns:
counts = df[c].value_counts()
if len(counts)!= 1:
ratios.append(counts[0]/counts[1])
 else:
toremove.append(c)
 print(f"remove {len(toremove)} columns")
 for t in toremove:
columns.remove(t)
 return columns, np.array(ratios ).astype(np.int32)
 columns, ratios = get_ratio_labels(train_targets_scored)
 columns_nonscored, ratios_nonscored = get_ratio_labels(train_targets_nonscored )<merge>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features
target = train[train_targets_scored.columns]<drop_column>
ind = ind.drop(columns = 'male' )
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<compute_train_metric>
range_ = lambda x: x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()
def evals(model, X, y, verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.clamp(y_preds, 0.0,1.0 ).detach().numpy()
score = log_loss_multi(y, y_preds)
 return y_preds, score
def inference_fn(model, X ,verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.sigmoid(torch.as_tensor(y_preds)).numpy()
return y_preds<compute_test_metric>
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()
p_min = 0.0005
p_max = 0.9995
def log_loss_score(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred)) 
<compute_test_metric>
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean() 
<compute_test_metric>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()<import_modules>
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False,pretrain=False, optimizer_params=None):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 if not pretrain:
self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() , **self.optimizer_params)
 else:
self.optimizer = self.optimizer_fn(self.network.parameters() , **optimizer_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res<import_modules>
features = list(train_set.columns)
 pipeline = Pipeline([('imputer', Imputer(strategy = 'median')) , 
('scaler', MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit
from sklearn.metrics import roc_auc_score<define_variables>
model = RandomForestClassifier(n_estimators=100, random_state=10,
n_jobs = -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
model.fit(train_set, train_labels)
 feature_importances = pd.DataFrame({'feature': features, 'importance': model.feature_importances_})
 feature_importances.head()
remove_vehicle= True<categorify>
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
def transform_data(train, test, col, normalize=True, removed_vehicle=False):

 mapping = {"cp_type":{"trt_cp": 0, "ctl_vehicle":1},
"cp_time":{48:0, 72:1, 24:2},
"cp_dose":{"D1":0, "D2":1}}
if removed_vehicle:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 else:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 max_ = 10.
 min_ = -10.
 if removed_vehicle:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
else:
numerical_tr = train[col[3:]]
numerical_test = test[col[3:]].values
if normalize:
numerical_tr =(numerical_tr-min_)/(max_ - min_)
 numerical_test =(numerical_test-min_)/(max_ - min_)
 return categories_tr, categories_test, numerical_tr, numerical_test
col_features = list(train_features)[1:]
cat_tr, cat_test, numerical_tr, numerical_test = transform_data(train_features, test_features, col_features, normalize=False, removed_vehicle=remove_vehicle)
 targets_tr = train_targets_scored[columns].values.astype(np.float32)
 targets2_tr = train_targets_nonscored[columns_nonscored].values.astype(np.float32 )<normalization>
model_results = cv_model(train_set, train_labels, LinearSVC() ,
'LSVC', model_results )
class LabelSmoothing(nn.Module):

 def __init__(self, smoothing=0.0, n_cls=2):

 super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing + smoothing / n_cls
self.smoothing = smoothing / n_cls
def forward(self, x, target):
probs = torch.nn.functional.sigmoid(x,)
 target1 = self.confidence * target +(1-target)* self.smoothing
loss = -(torch.log(probs+1e-15)* target1 +(1-target1)* torch.log(1-probs+1e-15))
return loss.mean()<init_hyperparams>
model_results = cv_model(train_set, train_labels,
GaussianNB() , 'GNB', model_results )
class Config(object):
def __init__(self):
self.num_class = 206
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = 7
self.EPOCHS = 200
self.num_ensembling = 6
self.seed = 0
self.cat_emb_dim= [5, 4]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals= numerical_tr.shape[1]
self.save_name = "tabnet_raw_step1"
self.strategy = "KFOLD"
cfg = Config() 
<concatenate>
model_results = cv_model(train_set, train_labels,
MLPClassifier(hidden_layer_sizes=(32, 64, 128, 64, 32)) ,
'MLP', model_results )
X_test = np.concatenate([cat_test, numerical_test ], axis=1 )<choose_model_class>
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )
<set_options>
model_results = cv_model(train_set, train_labels,
RidgeClassifierCV() , 'RIDGE', model_results )
warnings.filterwarnings("ignore" )<init_hyperparams>
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
for seed in range(cfg.num_ensembling):
seed_everything(seed)
 print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, lambda_sparse=0,
cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax', device_name=cfg.device,
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 print(X_train.shape, y_train.shape)
 print(X_val.shape, y_val.shape)
 model.fit(X_train=X_train, y_train=y_train, X_valid=X_val, y_valid=y_val,max_epochs=cfg.EPOCHS, patience=50,
batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=LabelSmoothing(0.001)
)
 model.load_best_model()
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.save_model(name)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )<compute_test_metric>
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))<split>
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(100, random_state=10),
'RF', model_results )
if cfg.strategy != "KFOLD":
i = 0
mskf = MultilabelStratifiedShuffleSplit(n_splits=1000, test_size=0.1, random_state=0)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
if i == cfg.SPLITS:
break
if not check_targets(targets_tr[train_idx]):
continue
print("FOLDS : ", i, j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_dims=cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=1e-3, amsgrad=True), mask_type="sparsemax", device_name=cfg.device)
 name = cfg.save_name + f"_{j}"
model.load_model(name)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 i+=1
p = np.stack(p)
 preds_test.append(p)
 preds_test = np.stack(preds_test)
<compute_test_metric>
def submit(model):
model.fit(train_set, train_labels)
 predictions = model.predict(test_set)
 predictions = pd.DataFrame({'idhogar': test_ids,
'Target': predictions})
 submission = submission_base.merge(predictions,
on = 'idhogar',
how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission
if cfg.strategy != "KFOLD":
print("auc mean : ", sum(scores_auc)/len(scores_auc))
print("CV score : ", log_loss_multi(np.concatenate(oof_targets), np.concatenate(oof_preds)) )<train_model>
X_train, X_test, y_train, y_test = train_test_split(train_set, train_labels, test_size = 0.2,
random_state =42 )
print(model )<feature_engineering>
import xgboost
from xgboost import XGBClassifier
from sklearn.metrics import f1_score
sample_submission[columns] = preds_test.mean(1 ).mean(0)
 sample_submission.loc[test_features['cp_type']=='ctl_vehicle', sample_submission.columns[1:]] = 0<save_to_csv>
xgb_model = XGBClassifier(objective = 'multi:softmax', random_state = 42)
 xgb_model.fit(X_train, y_train)
 y_pred = xgb_model.predict(X_test)
 f1_score(y_test, y_pred, average = 'macro' )
sample_submission.to_csv("submission.csv", index=False )<import_modules>
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'] )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook<load_from_csv>
import lightgbm
from lightgbm import LGBMClassifier
train=pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train<load_from_csv>
lgb_model = LGBMClassifier(objective = 'multiclass', random_state = 42)
 lgb_model.fit(X_train, y_train)
 y_pred = lgb_model.predict(X_test)
 print(y_pred)
 f1_score(y_test, y_pred, average = 'macro' )
train_target = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv")
 train_target<drop_column>
xgb_submission.to_csv('xgb.csv', index = False)
 lgb_submission.to_csv('lgb.csv', index = False )
ctlVehicle_idx = train["cp_type"] != "ctl_vehicle"
train = train.loc[ctlVehicle_idx].reset_index(drop=True)
 train = train.drop("cp_type", axis=1)
 train_target = train_target.loc[ctlVehicle_idx].reset_index(drop=True )<define_variables>
print('Train start')
 params = {"objective" : "multiclass",
"num_class" : 4,
"num_leaves" : 60,
"max_depth": -1,
"learning_rate" : 0.01,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
"bagging_freq" : 5,
"bagging_seed" : 42,
"verbosity" : -1 }
gcols = [g for g in train.columns if "g-" in g]
ccols = [c for c in train.columns if "c-" in c]
cpcols = [cp for cp in train.columns if "cp_" in cp]<load_from_csv>
y_tra = y_train - 1
y_tes = y_test -1
test = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 ctlVehicle_test = test["cp_type"] == "ctl_vehicle"
test = test.drop("cp_type", axis=1)
 train["cp_time"]=train["cp_time"].map({24: 1, 48: 2, 72: 3})
 train["cp_dose"] = train["cp_dose"].map({"D1": 1, "D2": 2})
 test["cp_time"]= test["cp_time"].map({24: 1, 48: 2, 72: 3})
 test["cp_dose"]=test["cp_dose"].map({"D1": 1, "D2": 2})
 train<drop_column>
lgtrain, lgval = lgb.Dataset(data = X_train, label = y_tra), lgb.Dataset(data = X_test, label = y_tes)
 print(lgtrain)
 lgb_ad_model = lgb.train(params, lgtrain, num_boost_round = 2000, valid_sets=[lgtrain, lgval],
early_stopping_rounds=100, verbose_eval=100 )
train_target = train_target.drop("sig_id", axis=1)
 train_target<drop_column>
predictions = lgb_ad_model.predict(test_set )
train = train.drop("sig_id", axis=1)
 test = test.drop("sig_id", axis=1 )<choose_model_class>
predict_df_1 = []
predict_df_2 = []
predict_df_3 = []
predict_df_4 = []
def create_shallow_model(out_shape=206):
inp = tf.keras.Input(shape=(train.shape[1],))
x = tfa.layers.WeightNormalization(L.Dense(train.shape[1]))(inp)
 x = L.BatchNormalization()(x)
 x = L.LeakyReLU(alpha=0.005 )(x)
 x = L.Dropout(0.2 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(128))(x)
 x = L.BatchNormalization()(x)
 x = L.LeakyReLU(alpha=0.005 )(x)
 x = L.Dropout(0.2 )(x)
 output = tfa.layers.WeightNormalization(L.Dense(out_shape, activation="sigmoid"))(x)
 model = tf.keras.Model(inputs=inp, outputs=output)
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=adamw)
 return model
def create_mid_model(out_shape=206):
inp = tf.keras.Input(shape=(train.shape[1],))
x = tfa.layers.WeightNormalization(L.Dense(train.shape[1]))(inp)
 x = L.BatchNormalization()(x)
 x = L.LeakyReLU(alpha=0.001 )(x)
 x = L.Dropout(0.3 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(512))(x)
 x = L.BatchNormalization()(x)
 x = L.LeakyReLU(alpha=0.001 )(x)
 x = L.Dropout(0.3 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(512))(x)
 x = L.BatchNormalization()(x)
 x = L.LeakyReLU(alpha=0.001 )(x)
 x = L.Dropout(0.2 )(x)
 output = tfa.layers.WeightNormalization(L.Dense(out_shape, activation="sigmoid"))(x)
 model = tf.keras.Model(inputs=inp, outputs=output)
 sgd = tf.keras.optimizers.SGD(momentum=0.9)
 adamw = tfa.optimizers.AdamW(weight_decay = 0.001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=radam)
 return model
def create_deep_model(out_shape=206):
inp = tf.keras.Input(shape=(train.shape[1],))
x = tfa.layers.WeightNormalization(L.Dense(train.shape[1]))(inp)
 x = L.BatchNormalization()(x)
 x = L.Activation("relu" )(x)
 x = L.Dropout(0.3 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(512))(x)
 x = L.BatchNormalization()(x)
 x = L.Activation("relu" )(x)
 x = L.Dropout(0.42 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(512))(x)
 x = L.BatchNormalization()(x)
 x = L.Activation("relu" )(x)
 x = L.Dropout(0.3 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(256))(x)
 x = L.BatchNormalization()(x)
 x = L.Activation("relu" )(x)
 x = L.Dropout(0.3 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(256))(x)
 x = L.BatchNormalization()(x)
 x = L.Activation("relu" )(x)
 x = L.Dropout(0.2 )(x)
 output = tfa.layers.WeightNormalization(L.Dense(out_shape, activation="sigmoid"))(x)
 model = tf.keras.Model(inputs=inp, outputs=output)
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=adamw)
 return model<train_on_grid>
for i in range(len(predictions)) :
predict_df_1.append(predictions[i][0])
 predict_df_2.append(predictions[i][1])
 predict_df_3.append(predictions[i][2])
 predict_df_4.append(predictions[i][3] )
predictions = []
kf = MultilabelStratifiedKFold(n_splits=5)
 tf.random.set_seed(42)
 for fold_id,(train_idx, valid_idx)in enumerate(kf.split(train, train_target)) :
model2 = create_mid_model()
history2 = model2.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=200,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model2_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 2, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history2.history["loss"]), min(history2.history["val_loss"])))
 model2 = create_mid_model()
model2.load_weights("model2_fold" + str(fold_id)+ ".h5")
 pred2 = model2.predict(test)
 predictions.append(pred2 )<prepare_output>
df_pred = pd.DataFrame(predict_df_1, columns = ['1'] )
pred = np.mean(predictions, axis=0)
 pred.shape<feature_engineering>
df_pred['2'] = predict_df_2
df_pred['3'] = predict_df_3
df_pred['4'] = predict_df_4
df_pred.head()
for i in range(len(pred)) :
for j in range(len(pred[0])) :
if pred[i][j] < 0.5:
pred[i][j] = 0.001
else:
pred[i][j] = 0.999
pred<load_from_csv>
df_pred['Target'] = df_pred[['1', '2', '3', '4']].idxmax(axis = 1 )
<save_to_csv><EOS> 
df_pred.to_csv('submission.csv', index = False )