Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<categorify>
!pip uninstall fastai transformers allennlp kornia tokenizers -y
!pip install.. /input/fastai-2012-wheel/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
!pip install.. /input/fastai-2012-wheel/tokenizers-0.8.1-cp37-cp37m-manylinux1_x86_64.whl
!pip install.. /input/fastai-2012-wheel/kornia-0.4.0-py2.py3-none-any.whl
!pip install.. /input/fastai-2012-wheel/fastcore-1.0.11-py3-none-any.whl
!pip install.. /input/fast-v2-offline/dataclasses-0.6-py3-none-any.whl
!pip install.. /input/fast-v2-offline/torchvision-0.7.0-cp37-cp37m-manylinux1_x86_64.whl
!pip install.. /input/fastai-2012-wheel/fastai-2.0.12-py3-none-any.whl
print('category1')
 words_dict = get_dict_with_split(X_test[:,7])
 print('dv_cat1')
 cat1d_vec_matrix = dv_cat1.transform(words_dict )<define_variables>
PATH = '.. /input/lish-moa'
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master' )
dv_cat1 = 0<load_pretrained>
%pylab inline
print('Fastai: ',fastai.__version__)
 print('Cuda available:', torch.cuda.is_available())
 if torch.cuda.is_available() :
device = torch.cuda.current_device()
torch.cuda.empty_cache()
print('Device:', device)
 else:
device = 'cpu'
plt.rcParams['figure.facecolor'] = 'w'
plt.rcParams['figure.dpi'] = 100
pd.options.display.max_colwidth = 100
pd.options.display.max_columns = 200
def set_random_seed(seed_value, use_cuda=True):
np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 random.seed(seed_value)
 if use_cuda:
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
def rank_gauss(x):
N = x.shape[0]
temp = x.argsort()
rank_x = temp.argsort() / N
rank_x -= rank_x.mean()
rank_x *= 2
efi_x = erfinv(rank_x)
 efi_x -= efi_x.mean()
return efi_x
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
def fe_stats(train, test, features_g, features_c):
print('Adding extra features')
 for df in train, test:
df['e-g_sum'] = df[features_g].sum(axis = 1)
 df['e-g_mean'] = df[features_g].mean(axis = 1)
 df['e-g_std'] = df[features_g].std(axis = 1)
 df['e-g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['e-g_skew'] = df[features_g].skew(axis = 1)
 df['e-c_sum'] = df[features_c].sum(axis = 1)
 df['e-c_mean'] = df[features_c].mean(axis = 1)
 df['e-c_std'] = df[features_c].std(axis = 1)
 df['e-c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['e-c_skew'] = df[features_c].skew(axis = 1)
 df['e-gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['e-gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['e-gc_std'] = df[features_g + features_c].std(axis = 1)
 df['e-gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['e-gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['e-c52_c42'] = df['c-52'] * df['c-42']
df['e-c13_c73'] = df['c-13'] * df['c-73']
df['e-c26_c13'] = df['c-23'] * df['c-13']
df['e-c33_c6'] = df['c-33'] * df['c-6']
df['e-c11_c55'] = df['c-11'] * df['c-55']
df['e-c38_c63'] = df['c-38'] * df['c-63']
df['e-c38_c94'] = df['c-38'] * df['c-94']
df['e-c13_c94'] = df['c-13'] * df['c-94']
df['e-c4_c52'] = df['c-4'] * df['c-52']
df['e-c4_c42'] = df['c-4'] * df['c-42']
df['e-c13_c38'] = df['c-13'] * df['c-38']
df['e-c55_c2'] = df['c-55'] * df['c-2']
df['e-c55_c4'] = df['c-55'] * df['c-4']
df['e-c4_c13'] = df['c-4'] * df['c-13']
df['e-c82_c42'] = df['c-82'] * df['c-42']
df['e-c66_c42'] = df['c-66'] * df['c-42']
df['e-c6_c38'] = df['c-6'] * df['c-38']
df['e-c2_c13'] = df['c-2'] * df['c-13']
df['e-c62_c42'] = df['c-62'] * df['c-42']
df['e-c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'e-{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'e-{feature}_squared'] = df[feature] ** 2
return train, test
def prepare_data(add_pca=False, gaussify=False, variance_threshold=None):
print('Preparing dataset')
 scored = pd.read_csv(f'{PATH}/train_targets_scored.csv' ).set_index('sig_id')
 y_names_scored = scored.columns.to_list()
nonscored = pd.read_csv(f'{PATH}/train_targets_nonscored.csv' ).set_index('sig_id')
 y_names_nonscored = nonscored.columns.to_list()
y_names = y_names_scored + y_names_nonscored
train_feat = pd.read_csv(f'{PATH}/train_features.csv' ).set_index('sig_id')
 cat_names = ['cp_type', 'cp_time', 'cp_dose']
feat_g = train_feat.filter(regex='g-')
 feat_c = train_feat.filter(regex='c-')
 feat_g_names = feat_g.columns.to_list()
feat_c_names = feat_c.columns.to_list()
test_feat = pd.read_csv(f'{PATH}/test_features.csv' ).set_index('sig_id')
 train_feat, test_feat = fe_stats(train_feat, test_feat, feat_g_names, feat_c_names)
 feat_e_names = train_feat.filter(regex='e-' ).columns.to_list()
cont_names = train_feat.filter(regex='g-|c-|e-' ).columns.to_list()
print(f'Cont names(n={len(cont_names)}): {cont_names}')
 feat = train_feat.append(test_feat)
 feat_cont = feat[cont_names]
feat_cat = feat[cat_names]
del feat
if variance_threshold is not None:
print(f'Variance filter {variance_threshold}')
 var_thresh = VarianceThreshold(threshold=variance_threshold)
 transf = var_thresh.fit_transform(feat_cont)
 feat_cont = pd.DataFrame(transf, index=feat_cont.index)\ 
.add_prefix('g-c-e-cont-')
 cont_names = feat_cont.columns.to_list()
feat_cat.loc[:, 'cp_type'] = feat_cat.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 feat_cat.loc[:, 'cp_dose'] = feat_cat.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 feat_cat.loc[:, 'cp_time'] = feat_cat.loc[:, 'cp_time'].map({24: 0, 48: 0.5, 72:1})
 print(len(cont_names))
if gaussify:
print('Gaussify')
 for col in tqdm(cont_names, desc='Gaussify'):
feat_cont.loc[:, col] = rank_gauss(feat_cont[col])
 if add_pca:
print('Add PCA features')
 pca_g = pd.DataFrame(PCA(200 ).fit_transform(feat_g), index=feat_g.index ).add_prefix('g-pca-')
 pca_c = pd.DataFrame(PCA(20 ).fit_transform(feat_c), index=feat_c.index ).add_prefix('c-pca-')
 cont_names.extend(pca_g.columns)
 cont_names.extend(pca_c.columns)
 feat = pd.concat([feat_cat, feat_cont, pca_g, pca_c], axis=1)
 else:
feat = pd.concat([feat_cat, feat_cont], axis=1)
 train = pd.merge(feat.loc[train_feat.index], scored, left_index=True, right_index=True)
 train = pd.merge(train, nonscored, left_index=True, right_index=True)
 test = feat.loc[test_feat.index]
return train, test, y_names, y_names_scored, cont_names, cat_names
dv_cat2 = pickle.load(open('.. /dv_cat2.dv', 'rb'))<categorify>
train, test, y_names, y_names_scored, cont_names, cat_names = \
prepare_data(add_pca=True, gaussify=True, variance_threshold=0.85 )
print('category2')
 words_dict = get_dict_with_split(X_test[:,8])
 print('dv_cat2')
 cat2d_vec_matrix = dv_cat2.transform(words_dict )<define_variables>
def sigmoid(x):
return 1/(1 + np.exp(-x))
def predict(learn, data):
dl = learn.dls.test_dl(data)
 pred, _ = learn.get_preds(dl=dl)
 pred_scored = pred[:,:].numpy()
return pred_scored
def create_submission(pred):
submission = pd.read_csv(f'{PATH}/sample_submission.csv', index_col=0)
 submission.loc[:, :] = pred
submission.loc[test['cp_type']==1, :] = 0
submission.to_csv('./submission.csv')
 return submission
dv_cat2 = 0<load_pretrained>
class NNet(nn.Module):
def __init__(self, input_dim, layers,
output_dim, sigmoid=False, ps=0,
act_func=nn.ReLU):
super().__init__()
if isinstance(ps, float)or isinstance(ps, int):
ps = [ps]*len(hidden_layer_sizes)
 layers = [input_dim] + layers
last_layer = nn.Linear(layers[-1], output_dim)
 self.layers =\
[nn.Sequential(nn.Dropout(ps_),
nn.utils.weight_norm(nn.Linear(input_, output_)) ,
act_func())
 for input_, output_, ps_ in
zip(layers, layers[1:], ps)] +\
[last_layer]
if sigmoid:
self.layers = self.layers + [nn.Sigmoid() ]
self.layers = nn.Sequential(*self.layers)
 def forward(self, cate_x, cont_x):
x = self.layers(cont_x)
 return x
dv_cat3 = pickle.load(open('.. /dv_cat3.dv', 'rb'))<categorify>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
print('category3')
 words_dict = get_dict_with_split(X_test[:,9])
 print('dv_cat3')
 cat3d_vec_matrix = dv_cat3.transform(words_dict )<define_variables>
def train_k_fold(train, test, n_folds=5, batch_size=None,
model=None, opt_func=Adam,
fit_pretrain=None,
fit_finetune=None,
use_MLSF=True, y_names=None,
y_names_scored=None,
model_params=None, seed=None):
set_random_seed(seed)
 if use_MLSF:
print('Using Stratified KFold(MLSF)')
 kfold = MultilabelStratifiedKFold(n_folds, shuffle=True, random_state=seed)
 progress = tqdm(enumerate(kfold.split(train, train[y_names])) ,
total=n_folds, desc='MLSF cross-validation')
 else:
print('Using KFold')
 kfold = KFold(n_folds, shuffle=True, random_state=seed)
 progress = tqdm(enumerate(kfold.split(train)) , total=n_folds,
desc='K-fold cross-validation')
 pred = None
for i,(ndx_train, ndx_valid)in progress:
model = NNet(**model_params)
 splits = [ndx_train, ndx_valid]
to = TabularPandas(train, y_names=y_names,
cat_names = None, device=device,
cont_names = cont_names+cat_names,
procs = [Categorify, FillMissing, Normalize],
splits = [list(ndx_train), list(ndx_valid)])
 dls = to.dataloaders(bs=batch_size)
 learn = Learner(dls, model, loss_func=SmoothBCEwLogits(smoothing=0.004),
opt_func=opt_func, cbs=CudaCallback, metrics=BCEWithLogitsLossFlat())
 print('Pretrain')
 fit_pretrain(learn)
 pretrained_model = learn.model
pretrained_model.layers = nn.Sequential(*list(learn.model.layers[:-1]),
nn.Sequential(LinBnDrop(model_params['layers'][-1],
model_params['layers'][-1]),
nn.Linear(model_params['layers'][-1],206)))
 to = TabularPandas(train, y_names=y_names_scored,
cat_names = None, device=device,
cont_names = cont_names+cat_names,
procs = [Categorify, FillMissing, Normalize],
splits = [list(ndx_train), list(ndx_valid)])
 dls = to.dataloaders(bs=batch_size)
 learn = Learner(dls, pretrained_model, loss_func=SmoothBCEwLogits(smoothing=0.004),
opt_func=Adam, cbs=CudaCallback, metrics=BCEWithLogitsLossFlat())
 learn.freeze_to(-1)
 print('Tune')
 fit_finetune(learn)
 valid_value = learn.validate() [0]
print(f'Seed={seed}; Fold={i}: Validation loss:{valid_value:5.4f}')
 _pred = predict(learn, test)
 if pred is None:
pred = _pred
else:
pred += _pred
pred /= n_folds
return pred
dv_cat3 = 0<load_pretrained>
train, test, y_names, y_names_scored, cont_names, cat_names = \
prepare_data(add_pca=True, gaussify=True )
<categorify><EOS> 
pred = None
pred_valid = None
n_folds= 5
n_rounds = 2
batch_size = 128
opt_func = Adam
model_params = \
dict(
 input_dim = len(cont_names+cat_names),
output_dim = len(y_names),
layers = [1500, 1250, 1000, 750],
ps = [0.5, 0.35, 0.3, 0.25],
act_func = nn.GELU, 
)
 def fit_pretrain(learn):
cbs = [
EarlyStoppingCallback(monitor='valid_loss', min_delta=1e-5, patience=10),
SaveModelCallback(name='model'),
]
learn.fit_one_cycle(100, 1e-2, cbs=cbs)
 learn.load('model')
 learn.fit_one_cycle(100, slice(1e-3), wd=0.1, cbs=cbs)
 learn.load('model')
 def fit_finetune(learn):
cbs = [
EarlyStoppingCallback(monitor='valid_loss', min_delta=1e-5, patience=10),
SaveModelCallback(name='model'),
]
learn.freeze_to(-1)
 learn.lr_find()
show()
learn.fit_one_cycle(20, slice(1e-4), wd=0.1, cbs=cbs)
 learn.recorder.plot_loss(with_valid=True)
 learn.load('model')
 learn.fit_one_cycle(100, slice(1e-5, 1e-4), wd=0.1, cbs=cbs)
 learn.recorder.plot_loss(with_valid=True)
 learn.load('model')
 learn.freeze_to(-2)
 learn.fit_one_cycle(100, slice(1e-6, 1e-5), wd=0.2, cbs=cbs)
 learn.recorder.plot_loss(with_valid=True)
 learn.load('model')
 show()
pred = None
pred_valid = None
pred_valid_collect = []
losses_valid = []
valid_true = train[y_names_scored].values.flatten()
for i in tqdm(range(n_rounds), desc='Rounds'):
_pred = train_k_fold(train, test, n_folds, batch_size=batch_size,
fit_pretrain=fit_pretrain, fit_finetune=fit_finetune,
opt_func=opt_func, model_params=model_params,
seed=i, y_names=y_names,
y_names_scored=y_names_scored)
 if pred is None:
pred = _pred
else:
pred += _pred
pred /= n_rounds
print(create_submission(sigmoid(pred)) )