Kaggle competition: Mechanisms of Action (MoA) Prediction
clf = load_model('neural_network_checkpoint_training.h5')
 threshold = 0.5
test_prediction = clf.predict(X_test.values ).flatten()
test_prediction[test_prediction <= threshold] = 0
test_prediction[test_prediction > threshold] = 1
test_prediction = pd.DataFrame(test_prediction, columns=['Survived'], dtype=np.int)
 prediction = pd.concat([test_ids, test_prediction], axis=1)
 prediction.to_csv('submission_nn.csv', index=False )<load_from_csv>
warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
train_data = pd.read_csv(".. /input/titanic/train.csv")
 train_data.head()<load_from_csv>
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
test_data = pd.read_csv(".. /input/titanic/test.csv")
 test_data.head()<count_missing_values>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
data_dt.isna().sum()<drop_column>
train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )
data_dt.drop(['Cabin'], axis=1, inplace=True)
 data_dt.head()<data_type_conversions>
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )
data_dt_proc = data_dt.copy()
print("Mediana de idade: {}".format(data_dt_proc.Age.median()))
data_dt_proc.Age.fillna(data_dt_proc.Age.median() , inplace=True )<data_type_conversions>
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x
data_dt_proc.Fare.fillna(data_dt_proc.Fare.median() , inplace=True )<count_missing_values>
cat_features = ['cp_time', 'cp_dose']
num_features = [c for c in train.columns if train.dtypes[c] != 'object']
num_features = [c for c in num_features if c not in cat_features]
num_features = [c for c in num_features if c not in target_cols]
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )
data_dt_proc.isna().sum()<drop_column>
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.5
lr=1e-2
weight_decay=1e-6
batch_size=32
epochs=20
num_features=num_features
cat_features=cat_features
target_cols=target_cols
data_dt_proc.drop(['Ticket', 'PassengerId'], axis=1, inplace=True )<feature_engineering>
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x
data_dt_proc['Title'] = data_dt_proc['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
data_dt_proc.head()<count_values>
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
data_dt_proc['Title'].value_counts() 
<count_values>
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr, weight_decay=cfg.weight_decay)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions
print(len(data_dt_proc['Title'].value_counts()))<count_values>
oof = np.zeros(( len(train), len(CFG.target_cols)))
 predictions = np.zeros(( len(test), len(CFG.target_cols)))
 SEED = [0, 1, 2]
for seed in SEED:
_oof, _predictions = run_kfold_nn(CFG,
train, test, folds,
num_features, cat_features, target,
device,
n_fold=5, seed=seed)
 oof += _oof / len(SEED)
 predictions += _predictions / len(SEED)
 score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"Seed Averaged CV score: {score}" )
Title_Dictionary = {
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Officer",
"the Countess":"Royalty",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Royalty",
"Dona" : "Mrs"
}
data_dt_proc['Title'] = data_dt_proc.Title.map(Title_Dictionary)
 data_dt_proc['Title'].value_counts()<feature_engineering>
train[target_cols] = oof
train[['sig_id']+target_cols].to_csv('oof.csv', index=False)
 test[target_cols] = predictions
test[['sig_id']+target_cols].to_csv('pred.csv', index=False )
data_dt_proc['FamilySize'] = data_dt_proc['Parch'] + data_dt_proc['SibSp'] + 1
data_dt_proc.head()<groupby>
result = train_targets_scored.drop(columns=target_cols)\ 
.merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = result[target_cols].values
score = 0
for i in range(y_true.shape[1]):
_score = log_loss(y_true[:,i], y_pred[:,i])
 score += _score / y_true.shape[1]
logger.info(f"Final result: {score}" )
<feature_engineering><EOS> 
sub = submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()