Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<categorify>
!git clone https://github.com/fastai/fastai.git
!mv fastai/fastai/ fastai/Fastai
sys.path.append('./fastai/' )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
%matplotlib inline
%reload_ext autoreload
%autoreload 2
SEED_VALUE = 42
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=SEED_VALUE )<concatenate>
np.set_printoptions(threshold=50, edgeitems=20)
 warnings.filterwarnings("ignore", category=sklearn.exceptions.UndefinedMetricWarning)
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=SEED_VALUE ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))<create_dataframe>
!cp.. /input/*csv.
 PATH='./'
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=SEED_VALUE ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))<create_dataframe>
train = pd.read_csv(f'{PATH}train.csv')
 test = pd.read_csv(f'{PATH}test.csv' )
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))<merge>
test.loc[test['rez_esc'] == 99.0 , 'rez_esc'] = 5
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )<drop_column>
train.drop(columns=[x for x in train.columns if 'SQB' in x or x == 'agesq'], inplace=True)
 test.drop(columns=[x for x in test.columns if 'SQB' in x or x == 'agesq'], inplace=True )
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<define_variables>
train.replace([np.inf, -np.inf], np.nan, inplace=True)
 test.replace([np.inf, -np.inf], np.nan, inplace=True )
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))<categorify>
def repalce_v18q1(x):
if x['v18q'] == 0:
return x['v18q']
else:
return x['v18q1']
train['v18q1'] = train.apply(lambda x : repalce_v18q1(x),axis=1)
 test['v18q1'] = test.apply(lambda x : repalce_v18q1(x),axis=1)
 train['v2a1'] = train['v2a1'].fillna(value=train['tipovivi3'])
 test['v2a1'] = test['v2a1'].fillna(value=test['tipovivi3'] )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
train['rez_esc'] = train.v18q1.fillna(0 ).astype(np.int32)
 test['rez_esc'] = test.v18q1.fillna(0 ).astype(np.int32 )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
train['meaneduc'] = train.v18q1.fillna(0 ).astype(np.float32)
 test['meaneduc'] = test.v18q1.fillna(0 ).astype(np.float32 )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
train['roof_waste_material'] = np.nan
test['roof_waste_material'] = np.nan
train['electricity_other'] = np.nan
test['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
train['roof_waste_material'] = train.apply(lambda x : fill_roof_exception(x),axis=1)
 test['roof_waste_material'] = test.apply(lambda x : fill_roof_exception(x),axis=1)
 train['electricity_other'] = train.apply(lambda x : fill_no_electricity(x),axis=1)
 test['electricity_other'] = test.apply(lambda x : fill_no_electricity(x),axis=1 )
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<choose_model_class>
for c in train.columns:
print(c, len(set(train[c])) )
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<categorify>
ignore = [x for x in train.columns if x == 'Target' or x == 'idhogar'] + ['edjefe', 'edjefa', 'Id']
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
train['escolari_age'] = train['escolari']/train['age']
test['escolari_age'] = test['escolari']/test['age']
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features<define_variables>
df_train = pd.DataFrame()
df_test = pd.DataFrame()
aggr_mean_list = ['rez_esc', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4',
'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7',
'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']
other_list = ['escolari', 'age', 'escolari_age']
for item in aggr_mean_list:
group_train_mean = train[item].groupby(train['idhogar'] ).mean()
group_test_mean = test[item].groupby(test['idhogar'] ).mean()
new_col = item + '_aggr_mean'
df_train[new_col] = group_train_mean
df_test[new_col] = group_test_mean
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_train = train[item].groupby(train['idhogar'] ).agg(function)
 group_test = test[item].groupby(test['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_train[new_col] = group_train
df_test[new_col] = group_test
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1<choose_model_class>
df_test = df_test.reset_index()
df_train = df_train.reset_index()
train = pd.merge(train, df_train, on='idhogar')
 test = pd.merge(test, df_test, on='idhogar')
 train.fillna(value=0, inplace=True)
 test.fillna(value=0, inplace=True )
model = Model(num_features, num_all_targets)
 model<define_search_model>
train.drop(columns=['parentesco' + str(i+2)for i in range(11)], inplace=True)
 ignore.append('parentesco1' )
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()<train_model>
def toCategorical(inData, columns, name):
inData[name] = np.zeros_like(len(inData))
for i, c in enumerate(columns):
inData.loc[:, name] +=(i+1)*inData.loc[:, c]
inData.drop(columns=columns, inplace=True )
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
wall_mat = ['paredblolad', 'paredzocalo', 'paredpreb', 'pareddes', 'paredmad', 'paredzinc', 'paredfibras', 'paredother']
toCategorical(train, wall_mat, 'wall_mat')
 toCategorical(test, wall_mat, 'wall_mat' )
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
floor_mat = ['pisomoscer', 'pisocemento', 'pisoother', 'pisonatur', 'pisonotiene', 'pisomadera']
toCategorical(train, floor_mat, 'floor_mat')
 toCategorical(test, floor_mat, 'floor_mat' )
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions<feature_engineering>
roof_mat = ['techozinc', 'techoentrepiso', 'techocane', 'techootro', 'roof_waste_material']
toCategorical(train, roof_mat, 'roof_mat')
 toCategorical(test, roof_mat, 'roof_mat' )
str(timedelta(seconds=time_diff))<compute_test_metric>
water_prov = ['abastaguadentro', 'abastaguafuera', 'abastaguano']
toCategorical(train, water_prov, 'water_prov')
 toCategorical(test, water_prov, 'water_prov' )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<save_to_csv>
elec_prov = ['public', 'planpri', 'noelec', 'coopele', 'electricity_other']
toCategorical(train, elec_prov, 'elec_prov')
 toCategorical(test, elec_prov, 'elec_prov' )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<install_modules>
toilet = ['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6']
toCategorical(train, toilet, 'toilet')
 toCategorical(test, toilet, 'toilet' )
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
cooking = ['energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4']
toCategorical(train, cooking, 'cooking')
 toCategorical(test, cooking, 'cooking' )
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN<set_options>
rubbish = ['elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6']
toCategorical(train, rubbish, 'rubbish')
 toCategorical(test, rubbish, 'rubbish' )
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<define_variables>
wall_quality = ['epared1', 'epared2', 'epared3']
toCategorical(train, wall_quality, 'wall_quality')
 toCategorical(test, wall_quality, 'wall_quality' )
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = ["PCA", "ICA", "FactorAnalysis"]
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"<load_from_csv>
roof_quality = ['etecho1', 'etecho2', 'etecho3']
toCategorical(train, roof_quality, 'roof_quality')
 toCategorical(test, roof_quality, 'roof_quality' )
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )<filter>
floor_quality = ['eviv1', 'eviv2', 'eviv3']
toCategorical(train, floor_quality, 'floor_quality')
 toCategorical(test, floor_quality, 'floor_quality' )
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )<define_variables>
gender = ['male', 'female']
toCategorical(train, gender, 'gender')
 toCategorical(test, gender, 'gender' )
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]<concatenate>
civil_status = ['estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7']
toCategorical(train, civil_status, 'civil_status')
 toCategorical(test, civil_status, 'civil_status' )
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]<normalization>
education = ['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']
toCategorical(train, education, 'education')
 toCategorical(test, education, 'education' )
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass<create_dataframe>
house_ownership = ['tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']
toCategorical(train, house_ownership, 'house_ownership')
 toCategorical(test, house_ownership, 'house_ownership' )
if "PCA" in decompo:
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 if "TSNE" in decompo :
print(b_, "TSNE")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = TSNE(n_components = ncompo_genes, n_iter=900, method='exact' ).fit_transform(data_all[GENES])
 pca_cells = TSNE(n_components = ncompo_cells, n_iter=900, method='exact' ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 if "ICA" in decompo:
print(b_, "ICA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = FastICA(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = FastICA(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"ica_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"ica_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 if "FactorAnalysis" in decompo:
print(b_, "FactorAnalysis")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = FactorAnalysis(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = FactorAnalysis(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"factor_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"factor_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
<categorify>
region = ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']
toCategorical(train, region, 'region')
 toCategorical(test, region, 'region' )
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )<feature_engineering>
area = ['area1', 'area2']
toCategorical(train, area, 'area')
 toCategorical(test, area, 'area' )
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )<load_pretrained>
for c in [x for x in train.columns if x not in ignore]:
print(c, len(set(train[c])) )
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )<load_pretrained>
cat_vars = [
'hacdor',
'hacapo',
'v14a',
'refrig',
'v18q',
'cielorazo',
'dis',
'computer',
'television',
'mobilephone',
'wall_mat',
'floor_mat',
'roof_mat',
'water_prov',
'elec_prov',
'toilet',
'cooking',
'rubbish',
'wall_quality',
'roof_quality',
'floor_quality',
'gender',
'civil_status',
'education',
'house_ownership',
'region',
'area']
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )<drop_column>
train.replace({'dependency': {'no': 0, 'yes': 1}}, inplace=True)
 test.replace({'dependency': {'no': 0, 'yes': 1}}, inplace=True )
features_to_drop = ["sig_id", "cp_type"]
data_all.drop(features_to_drop, axis = 1, inplace = True)
 try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )<choose_model_class>
contin_vars = [x for x in train.columns if x not in ignore and x not in cat_vars]
for c in [x for x in contin_vars]:
print(c, len(set(train[c])) )
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)<import_modules>
dep = 'Target'
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )<prepare_x_and_y>
test.index=test['Id']
test[dep] = 0
len(test )
scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 0, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<compute_test_metric>
for v in cat_vars:
train[v] = train[v].astype('category' ).cat.as_ordered()
apply_cats(test, train)
 for v in contin_vars:
train[v] = train[v].fillna(0 ).astype('float32')
 test[v] = test[v].fillna(0 ).astype('float32')
 train.head(2 )
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )<save_to_csv>
pretrain = train[train.parentesco1 == 0].copy()
train = train[train.parentesco1 == 1].copy()
train.reset_index(inplace=True)
 pretrain.reset_index(inplace=True)
 n = len(train)
 print(f'Pretraining on {len(pretrain)}, final training on {n} points' )
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()<import_modules>
df, y, nas, mapper = proc_df(train[cat_vars+contin_vars+[dep]], 'Target', do_scale=True )
import numpy as np
import pandas as pd<load_from_csv>
df_test, _, nas, mapper = proc_df(test[cat_vars+contin_vars+[dep]], 'Target', do_scale=True,
mapper=mapper, na_dict=nas )
df1 = pd.read_csv('.. /input/moa-blending/0.01832.csv')
 df2 = pd.read_csv('.. /input/mypredpublic/my_predictions_0000001.csv')
 df3 = pd.read_csv('.. /input/moa-blending/0.01833.csv')
<sort_values>
df_pre, yp, nas, mapper = proc_df(pretrain[cat_vars+contin_vars+[dep]], 'Target', do_scale=True,
mapper=mapper, na_dict=nas )
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<sort_values>
cat_sz = [(c, len(train[c].cat.categories)+1)for c in cat_vars]
emb_szs = [(c, min(50,(c+1)//2)) for _,c in cat_sz]
df1 = order_sub(df1)
 df2 = order_sub(df2)
 df3 = order_sub(df3 )<prepare_output>
[x for x in df_test.columns if x not in df.columns] , [x for x in df.columns if x not in df_test.columns]
BLEND=df1.copy()
BLEND.iloc[:,1:] = df1.iloc[:,1:]*0.6 +df2.iloc[:,1:]*0.2 +df3.iloc[:,1:]*0.2<define_variables>
def inv_y(a): return np.exp(a )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL )<load_from_csv>
def macro(y_pred, y_true):
y_pred = np.argmax(inv_y(y_pred), axis=1)
 f1 = sklearn.metrics.f1_score(y_true, y_pred, average='macro')
 return f1
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<load_from_zip>
def getPreModel(md):
return md.get_learner(emb_szs, len(df.columns)-len(cat_vars),
0.05, 5, [100], [0.5] )
!mkdir -p /tmp/pip/cache/
!cp.. /input/segmentation-models-zip-003/efficientnet_pytorch-0.4.0.xyz /tmp/pip/cache/efficientnet_pytorch-0.4.0.tar.gz
!cp.. /input/segmentation-models-zip-003/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz
!cp.. /input/segmentation-models-zip-003/segmentation_models_pytorch-0.0.3.xyz /tmp/pip/cache/segmentation_models_pytorch-0.0.3.tar.gz<install_modules>
def getModel(md):
return md.get_learner(emb_szs, len(df.columns)-len(cat_vars),
0.05, 5, [100, 100], [0.5, 0.5] )
!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch
!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch<install_modules>
def preTrainModel(trn_idx, val_idx):
trn_Housholds = train['idhogar'].iloc[trn_idx]
val_Housholds = train['idhogar'].iloc[val_idx]
ptrn_idx = pretrain[pretrain['idhogar'].isin(trn_Housholds)].index
pval_idx = pretrain[pretrain['idhogar'].isin(val_Housholds)].index
tmpDF = df_pre.copy()
tmpY = yp.copy()
maxN = np.sum(np.equal(4, yp[ptrn_idx]))
for c in [1,2,3]:
rows = pd.Series(np.equal(c, yp[ptrn_idx]), name='bools')
 n = np.sum(rows)
 nCopyMult = maxN//n
for j in range(nCopyMult):
tmpDF = tmpDF.append(df_pre.iloc[ptrn_idx][rows.values].copy() , ignore_index=True)
 tmpY = np.append(tmpY, yp[ptrn_idx][rows].copy())
 pmd = ColumnarModelData.from_data_frame(PATH, pval_idx, tmpDF, tmpY.astype(int), cat_flds=cat_vars, bs=16,
is_reg=False, is_multi=False)
 print('Pretraining model')
 m = getPreModel(pmd)
 m.fit(2e-3, 1, wds=1e-3, metrics=[macro], cycle_len=15,use_clr=(5,8), best_save_name='pre' )
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/senetunetmodelcode'
sys.path.append(package_path )<set_options>
def trainModel(trn_idx, val_idx):
tmpDF = df.copy()
tmpY = y.copy()
maxN = np.sum(np.equal(4, y[trn_idx]))
for c in [1,2,3]:
rows = pd.Series(np.equal(c, y[trn_idx]), name='bools')
 n = np.sum(rows)
 nCopyMult = maxN//n
for j in range(nCopyMult):
tmpDF = tmpDF.append(df.iloc[trn_idx][rows.values].copy() , ignore_index=True)
 tmpY = np.append(tmpY, y[trn_idx][rows].copy())
 md = ColumnarModelData.from_data_frame(PATH, val_idx, tmpDF, tmpY.astype(int), cat_flds=cat_vars, bs=16,
test_df=df_test, is_reg=False, is_multi=False)
 m = getModel(md)
 m.model.load_state_dict(torch.load(m.get_model_path('pre')) , strict=False)
 m.freeze_to(2)
 print('Training last layer')
 m.fit(8e-2,1,wds=1e-3,cycle_len=15,use_clr=(5,8), metrics=[macro], best_save_name='tmpbest')
 m.load('tmpbest')
 m.unfreeze()
m.bn_freeze(True)
 lr = 8e-3
print('Final training')
 m.fit(np.array([lr/9,lr/3,lr])/5, 1, wds=1e-3, metrics=[macro], cycle_len=15,use_clr=(5,8), best_save_name='best')
 m.load('best')
 return m
warnings.filterwarnings('ignore')
<load_pretrained>
%%time
nSplits = 10
skf = StratifiedKFold(nSplits, True, 1234)
 folds = skf.split(df, y)
 pred_test = []
valScore = 0
for i,(trn_idx, val_idx)in enumerate(folds):
print('________________________')
 print('Running fold', i)
 preTrainModel(trn_idx, val_idx)
 m = trainModel(trn_idx, val_idx)
 score = macro(*m.predict_with_targs())
 valScore += score
print('Fold', i, 'score:', score)
 pred_test.append(m.predict(True))
print('________________________
' )
unet_efficientnet = smp.Unet("efficientnet-b5", encoder_weights="imagenet", classes=4, activation="sigmoid" ).cuda()
ckpt_path = ".. /input/unet-efficientnet-baseline/unet_efficientnet_model.pth"
device = torch.device("cuda")
 unet_efficientnet.to(device)
 unet_efficientnet.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 unet_efficientnet.load_state_dict(state["state_dict"] )<load_pretrained>
print("
CV finished, mean validation score:", valScore/nSplits )
ckpt_path = ".. /input/senetmodels/senext50_30_epochs_high_threshold.pth"
device = torch.device("cuda")
 senext50_30_epochs_high_threshold = Unet('se_resnext50_32x4d', encoder_weights=None, classes=4, activation=None ).cuda()
senext50_30_epochs_high_threshold.to(device)
 senext50_30_epochs_high_threshold.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 senext50_30_epochs_high_threshold.load_state_dict(state["state_dict"] )<load_pretrained>
testClassPred = np.argmax(inv_y(np.mean(pred_test, axis=0)) , axis=1)
 testClassPred
unet_se_resnext50_32x4d = \
load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()<load_pretrained>
test['Target']=testClassPred
clf_resnet34 = load('/kaggle/input/severstalmodels/resnet34_classify.pth' ).cuda()<load_pretrained>
csv_fn=f'{PATH}sub.csv'
ckpt_path = "/kaggle/input/pytorch-multi-label-classification/efficientnet_b0_model.pth"
device = torch.device("cuda")
 clf_efficientnet_b0 = EfficientNet.from_pretrained('efficientnet-b0', num_classes=4 ).cuda()
clf_efficientnet_b0.to(device)
 clf_efficientnet_b0.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 clf_efficientnet_b0.load_state_dict(state["state_dict"] )<load_pretrained>
test[['Target']].to_csv(csv_fn, index=True )
<load_pretrained><EOS> 
!rm -rf fastai