Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<train_model>
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 warnings.filterwarnings('ignore')
train_pool = Pool(X,y)
 m = CatBoostClassifier(iterations=300,eval_metric="AUC", boosting_type = 'Ordered')
 m.fit(X,y,silent=True)
 y_pred1 = m.predict(X_test)
 m.best_score_<save_to_csv>
X_train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 y_train = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 X_test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submit = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = prediction
sub.to_csv("submission.csv", index=False )<save_to_csv>
def preprocess(df):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
train = preprocess(X_train)
 test = preprocess(X_test)
 del y_train['sig_id']
y_train = y_train.loc[train['cp_type']==0].reset_index(drop=True)
 train = train.loc[train['cp_type']==0].reset_index(drop=True )
sub1 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub1["target"] = y_pred1
sub1.to_csv("submission1.csv", index=False )<save_to_csv>
nfolds = 7
nstarts = 1
nepochs = 50
batch_size = 128
val_batch_size = batch_size * 4
ntargets = y_train.shape[1]
targets = [col for col in y_train.columns]
criterion = nn.BCELoss()
kfold = MultilabelStratifiedKFold(n_splits=nfolds, random_state=517, shuffle=True)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 train = train.values
test = test.values
y_train = y_train.values
sub2 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub2["target"] =(prediction + y_pred1)/2
sub2.to_csv("submission2.csv", index=False )<load_from_csv>
class Model(nn.Module):
def __init__(self, num_columns):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_columns)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_columns, 2048))
self.batch_norm2 = nn.BatchNorm1d(2048)
 self.dropout2 = nn.Dropout(0.5)
 self.dense2 = nn.utils.weight_norm(nn.Linear(2048, 1024))
self.batch_norm3 = nn.BatchNorm1d(1024)
 self.dropout3 = nn.Dropout(0.5)
 self.dense3 = nn.utils.weight_norm(nn.Linear(1024, 206))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.sigmoid(self.dense3(x))
return x
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<drop_column>
class Dataset(Dataset):
def __init__(self, df, targets, mode='train'):
self.mode = mode
self.data = df
if mode=='train':
self.targets = targets
def __getitem__(self, idx):
if self.mode == 'train':
return torch.FloatTensor(self.data[idx]), torch.FloatTensor(self.targets[idx])
 elif self.mode == 'test':
return torch.FloatTensor(self.data[idx]), 0
def __len__(self):
return len(self.data )
train_features = train_df.drop(['target','ID_code'], axis = 1)
 test_features = test_df.drop(['ID_code'],axis = 1)
 train_target = train_df['target']<normalization>
for n,(tr, te)in enumerate(kfold.split(y_train, y_train)) :
print(f'Train fold {n+1}')
 xtrain, xval = train[tr], train[te]
ytrain, yval = y_train[tr], y_train[te]
train_set = Dataset(xtrain, ytrain)
 val_set = Dataset(xval, yval)
 dataloaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True),
'val': DataLoader(val_set, batch_size=val_batch_size, shuffle=False)
 }
model = Model(X_train.shape[1]-1 ).to(device)
 checkpoint_path = f'repeat:{1}_Fold:{n+1}.pt'
optimizer = optim.Adam(model.parameters() , weight_decay=1e-5)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4, verbose=True)
 best_loss = {'train': np.inf, 'val': np.inf}
for epoch in range(nepochs):
epoch_loss = {'train': 0.0, 'val': 0.0}
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
for i,(x, y)in enumerate(dataloaders[phase]):
x, y = x.to(device), y.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase=='train'):
preds = model(x)
 loss = criterion(preds, y)
 if phase=='train':
loss.backward()
optimizer.step()
running_loss += loss.item() / len(dataloaders[phase])
 epoch_loss[phase] = running_loss
print("Epoch {}/{} - loss: {:5.5f} - val_loss: {:5.5f}".format(epoch+1, nepochs, epoch_loss['train'], epoch_loss['val']))
scheduler.step(epoch_loss['val'])
 if epoch_loss['val'] < best_loss['val']:
best_loss = epoch_loss
torch.save(model.state_dict() , checkpoint_path )
sc = StandardScaler()
train_features = sc.fit_transform(train_features)
 test_features = sc.transform(test_features )<train_model>
oof = np.zeros(( len(train), nstarts, ntargets))
oof_targets = np.zeros(( len(train), ntargets))
preds = np.zeros(( len(test), ntargets))
n_splits = 5
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True ).split(train_features, train_target))
splits[:3]<choose_model_class>
def mean_log_loss(y_true, y_pred):
metrics = []
for i, target in enumerate(targets):
metrics.append(log_loss(y_true[:, i], y_pred[:, i].astype(float), labels=[0,1]))
return np.mean(metrics )
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<choose_model_class>
seed_targets = []
seed_oof = []
seed_preds = np.zeros(( len(test), ntargets, nfolds))
for n,(tr, te)in enumerate(kfold.split(y_train, y_train)) :
xval, yval = train[te], y_train[te]
fold_preds = []
val_set = Dataset(xval, yval)
 test_set = Dataset(test, None, mode='test')
 dataloaders = {
'val': DataLoader(val_set, batch_size=val_batch_size, shuffle=False),
'test': DataLoader(test_set, batch_size=val_batch_size, shuffle=False)
 }
checkpoint_path = f'repeat:{1}_Fold:{n+1}.pt'
model = Model(X_train.shape[1]-1 ).to(device)
 model.load_state_dict(torch.load(checkpoint_path))
model.eval()
for phase in ['val', 'test']:
for i,(x, y)in enumerate(dataloaders[phase]):
if phase == 'val':
x, y = x.to(device), y.to(device)
 elif phase == 'test':
x = x.to(device)
 with torch.no_grad() :
batch_preds = model(x)
 if phase == 'val':
seed_targets.append(y)
 seed_oof.append(batch_preds)
 elif phase == 'test':
fold_preds.append(batch_preds)
 fold_preds = torch.cat(fold_preds, dim=0 ).cpu().numpy()
seed_preds[:, :, n] = fold_preds
seed_targets = torch.cat(seed_targets, dim=0 ).cpu().numpy()
seed_oof = torch.cat(seed_oof, dim=0 ).cpu().numpy()
seed_preds = np.mean(seed_preds, axis=2)
 oof_targets = seed_targets
oof[:, 0, :] = seed_oof
preds += seed_preds / nstarts
oof = np.mean(oof, axis=1)
 print("Overall score is {:5.5f}".format(mean_log_loss(oof_targets, oof)) )
<compute_test_metric><EOS> 
submit[targets] = preds
submit.loc[X_test['cp_type']=='ctl_vehicle', targets] = 0
submit.to_csv('submission.csv', index=False )