Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<set_options>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
del embeddings_google
gc.collect()<compute_test_metric>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
mean_emb = np.mean([embedding_matrix, embedding_matrix_wiki, embedding_matrix_google], axis=0 )<data_type_conversions>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
char_embedding_matrix = np.random.randn(len(t2.word_index)+1,128 ).astype(np.float32)* np.sqrt(2.0/len(t2.word_index)+1 )<choose_model_class>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.9
decompo = "PCA"
ncompo_genes = 0.95
ncompo_cells = 0.95
encoding = "dummy"
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )<compute_test_metric>
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<set_options>
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )
class Configuration:
def get_config_object() :
config = Configuration()
config.max_seq_size = 90
config.word_vocab_size = vocab_size
config.embedding_dim = 300
config.units1 =64
config.units2 = 32
config.units3 = 32
return config<choose_model_class>
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
class Model:
def __init__(self, config):
self.config = config
def get_model(self):
input_layer = Input(batch_shape=(None,self.config.max_seq_size), name='input_layer')
 input_layer_2 = Input(batch_shape=(None,180), name='input_layer_cts')
 input_layer_3 = Input(batch_shape=(None,10), name='input_layer_feat')
 lda_input = Input(batch_shape=(None, 8), name='lda_input_layer')
 embed_op = Embedding(input_dim= self.config.word_vocab_size, input_length=self.config.max_seq_size, output_dim=self.config.embedding_dim,
weights=[mean_emb], trainable=True, name='word_embeddings' )(input_layer)
 spdropout = SpatialDropout1D(0.2, name='sp_dropout' )(embed_op)
 bigru1 = Bidirectional(CuDNNLSTM(units=self.config.units1, return_sequences=True, name='bigru1'))(spdropout)
 embed_op_2 = Embedding(input_dim=len(t2.word_index)+1, input_length=180, output_dim=128, name='char_embeddings' )(input_layer_2)
 spdropout_2 = SpatialDropout1D(0.2, name='sp_dropout_2' )(embed_op_2)
 conv1 = Conv1D(filters=32, kernel_size=2, name='conv1' )(spdropout_2)
 maxpool1 = GlobalMaxPool1D()(conv1)
 conv2 = Conv1D(filters=32, kernel_size=4, name='conv2' )(spdropout_2)
 maxpool2 = GlobalMaxPool1D()(conv2)
 conv3 = Conv1D(filters=32, kernel_size=8, name='conv3' )(spdropout_2)
 maxpool3 = GlobalMaxPool1D()(conv3)
 tile_layer = RepeatVector(n=90, name='tile_layer')
 tiled_op = tile_layer(lda_input)
 concat_1 = Concatenate(axis=2, name='concat_1' )([bigru1,tiled_op])
 att_dense_layer = Dense(units=1, name='attention_dense_1', activation='tanh')
 att_dense_op_1 = att_dense_layer(concat_1)
 softmax_layer = Softmax(axis=1, name='softmax_over_time')
 alpha_1 = softmax_layer(att_dense_op_1)
 dot_layer = Dot(axes=1, name='dot_layer')
 context_1 = dot_layer([alpha_1, concat_1])
 context_reshape_1 = Reshape(target_shape=(136,), name='reshape_context' )(context_1)
 selector = Lambda(lambda x: x[:, -1], name='slicer')
 lt_bigru1 = selector(bigru1)
 concat = concatenate([lt_bigru1,context_reshape_1,maxpool1,maxpool2, maxpool3,lda_input, input_layer_3], name='concatenate')
 dense1 = Dense(units=self.config.units3, activation='tanh', name='dense1' )(concat)
 drop1 = GaussianDropout(0.1, name='Dropout_1_c3' )(dense1)
 bn = BatchNormalization()(drop1)
 dense2 = Dense(units=16, activation='tanh', name='dense2' )(bn)
 dense3 = Dense(units=1, activation='sigmoid', name='output' )(dense2)
 classifier = keras.models.Model(inputs=[input_layer,input_layer_2,input_layer_3,lda_input], outputs=dense3)
 return classifier
def get_model_2(self):
input_layer = Input(batch_shape=(None,self.config.max_seq_size), name='input_layer')
 input_layer_2 = Input(batch_shape=(None,180), name='input_layer_cts')
 input_layer_3 = Input(batch_shape=(None,10), name='input_layer_feat')
 lda_input = Input(batch_shape=(None, 8), name='lda_input_layer')
 embed_op1 = Embedding(input_dim= self.config.word_vocab_size, input_length=90, output_dim=self.config.embedding_dim,
weights=[embedding_matrix], trainable=True, name='word_embeddings' )(input_layer)
 spdropout1 = SpatialDropout1D(0.2, name='sp_dropout1' )(embed_op1)
 conv1_word = Conv1D(filters=32, kernel_size=2, name='conv1_word' )(spdropout1)
 maxpool1_word = GlobalMaxPool1D()(conv1_word)
 avgpool1_word = GlobalAvgPool1D()(conv1_word)
 conv2_word = Conv1D(filters=32, kernel_size=4, name='conv2_word' )(spdropout1)
 maxpool2_word = GlobalMaxPool1D()(conv2_word)
 avgpool2_word = GlobalAvgPool1D()(conv2_word)
 conv3_word = Conv1D(filters=32, kernel_size=8, name='conv3_word' )(spdropout1)
 maxpool3_word = GlobalMaxPool1D()(conv3_word)
 avgpool3_word = GlobalAvgPool1D()(conv3_word)
 embed_op_2 = Embedding(input_dim=len(t2.word_index)+1, input_length=180, output_dim=128, name='char_embeddings' )(input_layer_2)
 spdropout_2 = SpatialDropout1D(0.2, name='sp_dropout_2' )(embed_op_2)
 conv1 = Conv1D(filters=32, kernel_size=2, name='conv1' )(spdropout_2)
 maxpool1 = GlobalMaxPool1D()(conv1)
 avgpool1 = GlobalAvgPool1D()(conv1)
 conv2 = Conv1D(filters=32, kernel_size=4, name='conv2' )(spdropout_2)
 maxpool2 = GlobalMaxPool1D()(conv2)
 avgpool2 = GlobalAvgPool1D()(conv2)
 conv3 = Conv1D(filters=32, kernel_size=8, name='conv3' )(spdropout_2)
 maxpool3 = GlobalMaxPool1D()(conv3)
 avgpool3 = GlobalAvgPool1D()(conv3)
 concat = concatenate([maxpool1_word,maxpool2_word, maxpool3_word,avgpool1_word, avgpool2_word, avgpool3_word,maxpool1,maxpool2, maxpool3,avgpool1, avgpool2, avgpool3,lda_input, input_layer_3], name='concatenate')
 dense1 = Dense(units=self.config.units3, activation='tanh', name='dense1' )(concat)
 drop1 = GaussianDropout(0.1, name='Dropout_1_c3' )(dense1)
 bn = BatchNormalization()(drop1)
 dense2 = Dense(units=16, activation='tanh', name='dense2' )(bn)
 dense3 = Dense(units=1, activation='sigmoid', name='output' )(dense2)
 classifier = keras.models.Model(inputs=[input_layer,input_layer_2,input_layer_3,lda_input], outputs=dense3)
 return classifier<choose_model_class>
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
config = Configuration.get_config_object()
model = Model(config)
 opt = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True )<choose_model_class>
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass
classifier = model.get_model()
classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', precision, recall, f1])
 classifier.summary()<choose_model_class>
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )
earlystopping = EarlyStopping(monitor='val_f1', patience=1, mode='max')
 modelcheckpoint = ModelCheckpoint(filepath='model.h5', monitor='val_f1', verbose=1, save_best_only=True, mode='max')
 pbar = TQDMNotebookCallback(leave_inner=True, leave_outer=True)
 DATA_SPLIT_SEED = 2018
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300, mode='exp_range',
gamma=0.99994)
 callbacks1 = [earlystopping,modelcheckpoint, clr]<train_model>
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
relation_cols = []
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1)
 relation_cols.append("g_" + stats)
 relation_cols.append("c_" + stats)
 relation_cols.append("gc_" + stats )
cw = {0:2, 1: 2.6}
classifier.fit([X_train_tts,X_train_cts,feat_train ,ldavec_train], y_train, callbacks=callbacks1 ,validation_data=([X_dev_tts,X_dev_cts,feat_dev,ldavec_dev], y_dev), epochs=1, batch_size=128, verbose=1, class_weight=cw )<train_model>
categorical_cols = [cc for cc in data_all.columns if "cp_" in cc]
categorical_cols.remove("cp_type")
 categorical_cols
cw = {0:1, 1: 1.6}
classifier.fit([X_train_tts,X_train_cts,feat_train ,ldavec_train], y_train, callbacks=callbacks1 ,validation_data=([X_dev_tts,X_dev_cts,feat_dev,ldavec_dev], y_dev), epochs=1, batch_size=256, verbose=1, class_weight=cw )<predict_on_test>
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = joblib.load(".. /input/moa-pca-joblib-myjob/genes.joblib")
 pca_cells = joblib.load(".. /input/moa-pca-joblib-myjob/cells.joblib")
 g_pca = [f"pca_g-{i}" for i in range(pca_genes.n_components_)]
c_pca = [f"pca_c-{i}" for i in range(pca_cells.n_components_)]
pca_genes_df = pd.DataFrame(pca_genes.transform(data_all[GENES]), columns = g_pca)
 pca_cells_df = pd.DataFrame(pca_cells.transform(data_all[CELLS]), columns = c_pca)
 pca_cols = g_pca + c_pca
data_all = pd.concat([pca_genes_df, pca_cells_df, data_all[cols_numeric + categorical_cols + relation_cols]], axis = 1)
 else:
pass
pred = classifier.predict([test_tts,test_cts,feat_test,ldavec_test], verbose=1 )<data_type_conversions>
GENES_PCA = [g for g in data_all.columns if "pca_g-" in g]
CELLS_PCA = [c for c in data_all.columns if "pca_c-" in c]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["pca_g-" + stats] = getattr(data_all[GENES_PCA], stats )(axis = 1)
 data_all["pca_c-" + stats] = getattr(data_all[CELLS_PCA], stats )(axis = 1)
 data_all["pca_gc-" + stats] = getattr(data_all[GENES_PCA + CELLS_PCA], stats )(axis = 1 )
pred =(pred > 0.5 ).astype(int )<feature_engineering>
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )
test['prediction'] = pred<drop_column>
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )
final = test.drop(['question_text','feat_vect','fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm', 'clean','clean_2'], axis=1 )<save_to_csv>
try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )
final.to_csv('submission.csv',index=False )<import_modules>
cat_idx = []
for col in categorical_cols:
cat_idx.append(data_all.columns.get_loc(col))
cat_idx
stopwords = stopwords.words('english')
<load_from_csv>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-5)+ y_true * np.log(logits + 1e-5)
 return np.mean(-aux )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print('Train shape : ',train.shape[0])
 print('Test shape : ',test.shape[0] )<split>
MAX_EPOCH = 300
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10,
cat_idxs = cat_idx 
)
 scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 7
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 678234, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embed_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))<feature_engineering>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
<define_variables><EOS> 
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 for col in cols_to_zero:
submission[col] = [0.000012] * len(submission)
 submission.to_csv("submission.csv", index = None)
 submission.head()