Kaggle competition: Mechanisms of Action (MoA) Prediction
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=0, stratify=y)
 print("X_train dataset: ", X_train.shape)
 print("y_train dataset: ", y_train.shape)
 print("X_valid dataset: ", X_valid.shape)
 print("y_valid dataset: ", y_valid.shape )<create_dataframe>
GENES_REGEX = '^g-*|^c-'
N_SPLITS = 3
BATCH_SIZE=128
EPOCHS=200
LEARNING_RATE = 4e-1
LABEL_SMOOTHING = 1e-5
RLR_PATIENCE = 4
RLR_FACTOR =.7
ES_PATIENCE = 20
params = {
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.04,
'learning_rate': 0.008,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
lg_train = lgb.Dataset(X_train, label= y_train.ravel())
 lg_valid = lgb.Dataset(X_valid, label=y_valid.ravel())
 model = lgb.train(params, lg_train, 10000, valid_sets=[lg_valid],
early_stopping_rounds=1000, verbose_eval=1000)
<predict_on_test>
REGULARIZATION = 1
INPUT_SHAPE = 729
AUTOCOORD_L1, AUTOCOORD_L2 = 1e-3, 1e-2
AUTO_SIZE = 9**3
RESHAPE_SIZE =(9,9,9,1)
 AUTO_DROPOUT = 0
CONV_FILTER = 2
CONV_DROPOUT =.2
RESBLOCK_L1, RESBLOCK_L2 = 1e-12, 1e-11
DENSE_L1, DENSE_L2 = 1e-9, 1e-8
DENSE_DROPOUT =.4
OUTPUT_SIZE = 206
pred_val_lgb = model.predict(X_valid, num_iteration=model.best_iteration )<predict_on_test>
def head(numpy_array):
return pd.DataFrame(numpy_array)
 train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_features.head()
genes = train_features.filter(regex=(GENES_REGEX)).to_numpy()
head(genes)
 targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop(['sig_id'],axis=1 ).to_numpy()
non_scored_targets = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv' ).drop(['sig_id'],axis=1 ).to_numpy()
drug_targets = pd.get_dummies(pd.read_csv('.. /input/lish-moa/train_drug.csv' ).drop(['sig_id'],axis=1)).to_numpy()
all_targets = np.concatenate([targets, non_scored_targets], axis=-1)
 train_features.head()
pred_test_lgb = model.predict(test_df_new.values, num_iteration=model.best_iteration )<save_to_csv>
def cleanFeatures(features):
features_ = features.copy()
cp_type = {'trt_cp': 0, 'ctl_vehicle': 1}
cp_dose = {'D1': 0, 'D2': 1}
features_['cp_type'] = features_['cp_type'].map(cp_type)
 features_['cp_dose'] = features_['cp_dose'].map(cp_dose)
 features_ = pd.get_dummies(features_, columns=['cp_time'])
 features_.drop(['sig_id'], inplace=True, axis=1)
 return features_
X_clean = cleanFeatures(train_features)
 X_clean
sub_df_lgb = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_lgb["target"] = pred_test_lgb
sub_df_lgb.to_csv("lgb_submission.csv", index=False)
<choose_model_class>
def rankGauss(x):
N = x.shape[0]
temp = x.argsort(axis=1)
 rank_x = temp.argsort(axis=1)/ N
rank_x -= rank_x.mean()
rank_x *= 2
efi_x = erfinv(rank_x)
 efi_x -= efi_x.mean()
return efi_x
xgb_model = xgb.XGBClassifier(max_depth=2, learning_rate=0.02, n_estimators=3000, booster="gbtree",
objective="binary:logistic", min_child_weight=1, subsample = 0.8,
colsample_bytree = 0.6, random_state=1234 )<train_model>
REGEX = '^cp_*'
X = rankGauss(genes)
 other_X = X_clean.filter(regex=(REGEX)).to_numpy()
X.shape, other_X.shape
xgb_model.fit(X_train, y_train.ravel() , eval_set=[(X_valid, y_valid.ravel())], eval_metric="auc",
early_stopping_rounds=1000, verbose=True )<predict_on_test>
pred_test_xgb = xgb_model.predict_proba(test_df_new.values)[:, 1]<predict_on_test>
copyfile(src = ".. /input/minimalistic-v2/ml_stratifiers.py", dst = ".. /working/ml_stratifiers.py")
 def getMskfDataset(X, other_X, y, all_y, n_splits=N_SPLITS, SEED=7):
mskf = MultilabelStratifiedKFold(n_splits=n_splits, random_state=SEED, shuffle=True)
 for i,(train_idx, val_idx)in enumerate(mskf.split(X, y)) :
X_train, other_X_train, y_train = X[train_idx], other_X[train_idx], y[train_idx]
X_val, other_X_val, y_val = X[val_idx], other_X[val_idx], y[val_idx]
X_train_0, other_X_train_0, y_train_0 = X[train_idx], other_X[train_idx], all_y[train_idx]
X_val_0, other_X_val_0, y_val_0 = X[val_idx], other_X[val_idx], all_y[val_idx]
X_train_1, other_X_train_1, y_train_1 = X[train_idx], other_X[train_idx], y[train_idx]
X_val_1, other_X_val_1, y_val_1 = X[val_idx], other_X[val_idx], y[val_idx]
yield X_train_0, other_X_train_0, y_train_0, X_val_0, other_X_val_0, y_val_0, X_train_1, other_X_train_1, y_train_1, X_val_1, other_X_val_1, y_val_1
pred_test_xgb_new = xgb_model.predict(test_df_new.values )<save_to_csv>
sub_df_xgb = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_xgb["target"] = pred_test_xgb
sub_df_xgb.to_csv("xgb_submission.csv", index=False)
 sub_df_xgb.head()<choose_model_class>
class AutoCoordBlock(tf.keras.layers.Layer):
def __init__(self, dense_size=AUTO_SIZE, trainable=True):
super(AutoCoordBlock, self ).__init__()
self.dense_size= dense_size
self.l2 = tf.keras.regularizers.L2(l2=AUTOCOORD_L1)
 self.l1 = tf.keras.regularizers.L1(l1=AUTOCOORD_L2*1e-7)
 self.dense = tf.keras.layers.Dense(self.dense_size, kernel_regularizer=self.l2 ,bias_regularizer=self.l1, trainable=trainable)
 self.batch_norm = tf.keras.layers.BatchNormalization(trainable=trainable)
 self.relu = tf.keras.layers.ReLU()
self.dropout = tf.keras.layers.Dropout(AUTO_DROPOUT)
 self.reshape= tf.keras.layers.Reshape(RESHAPE_SIZE)
 @tf.function
def call(self, x, training=False):
x = self.dense(x, training=training)
 x = self.batch_norm(x,training=training)
 x = self.relu(x, training=training)
 x = self.reshape(x, training=training)
 x = self.dropout(x, training=training)
 return x
class ResBlock(tf.keras.layers.Layer):
def __init__(self, pre_filter_size, k3_filter_size, k1_filter_size, padding='same', is_first=False, trainable=True):
super(ResBlock, self ).__init__()
self.padding = padding
self.l2 = tf.keras.regularizers.L2(l2=RESBLOCK_L1)
 self.l1 = tf.keras.regularizers.L1(l1=RESBLOCK_L2)
 self.is_first = is_first
self.dropout = tf.keras.layers.Dropout(CONV_DROPOUT)
 if not is_first:
self.pre_filter_size = pre_filter_size
self.pre_conv = tf.keras.layers.Conv3D(self.pre_filter_size,(1,1,1), padding=self.padding, kernel_regularizer=self.l2 ,bias_regularizer=self.l1, trainable=trainable)
 self.batch_norm_pre = tf.keras.layers.BatchNormalization(trainable=trainable)
 self.relu_pre = tf.keras.layers.ReLU()
self.k1_filter_size = k1_filter_size
self.conv_k1 = tf.keras.layers.Conv3D(self.k1_filter_size,(5,5,5), padding=self.padding, kernel_regularizer=self.l2 ,bias_regularizer=self.l1, trainable=trainable)
 self.batch_norm_k1 = tf.keras.layers.BatchNormalization(trainable=trainable)
 self.relu_k1 = tf.keras.layers.ReLU()
self.k3_filter_size = k3_filter_size
self.conv_k3 = tf.keras.layers.Conv3D(self.k3_filter_size,(3,3,3), padding=self.padding, kernel_regularizer=self.l2 ,bias_regularizer=self.l1, trainable=trainable)
 self.batch_norm_k3 = tf.keras.layers.BatchNormalization(trainable=trainable)
 self.relu_k3 = tf.keras.layers.ReLU()
self.concat = tf.keras.layers.Concatenate(axis=-1)
 @tf.function
def call(self, x, training=False):
if not self.is_first:
x = self.pre_conv(x, training=training)
 x = self.batch_norm_pre(x,training=training)
 x = self.relu_pre(x, training=training)
 x = self.dropout(x, training = training)
 x_k1 = self.conv_k1(x, training=training)
 x_k1 = self.batch_norm_k1(x_k1, training=training)
 x_k1 = self.relu_k1(x_k1, training=training)
 x_k1 = self.dropout(x, training = training)
 x_k3 = self.conv_k3(x, training=training)
 x_k3 = self.batch_norm_k3(x_k3,training=training)
 x_k3 = self.relu_k3(x_k3, training=training)
 x_k3 = self.dropout(x, training = training)
 if not self.is_first:
x_ = self.concat([x_k3, x_k1, x])
 else:
x_ = self.concat([x_k3, x])
 return x_
class ReduceBlock(tf.keras.layers.Layer):
def __init__(self, trainable=True):
super(ReduceBlock, self ).__init__()
self.l2 = tf.keras.regularizers.L2(l2=RESBLOCK_L1)
 self.l1 = tf.keras.regularizers.L1(l1=RESBLOCK_L2)
 self.padding = 'valid'
self.flatten = tf.keras.layers.Flatten()
self.dropout = tf.keras.layers.Dropout(CONV_DROPOUT)
 self.filter_size_0 = 8*CONV_FILTER
self.conv_0 = tf.keras.layers.Conv3D(self.filter_size_0,(3,3,3), padding=self.padding, kernel_regularizer=self.l2 ,bias_regularizer=self.l1, trainable=trainable)
 self.batch_norm_0 = tf.keras.layers.BatchNormalization(trainable=trainable)
 self.relu_0 = tf.keras.layers.ReLU()
self.filter_size_1 = 16*CONV_FILTER
self.conv_1 = tf.keras.layers.Conv3D(self.filter_size_1,(3,3,3), padding=self.padding, kernel_regularizer=self.l2 ,bias_regularizer=self.l1, trainable=trainable)
 self.batch_norm_1 = tf.keras.layers.BatchNormalization(trainable=trainable)
 self.relu_1 = tf.keras.layers.ReLU()
self.filter_size_2 = 32*CONV_FILTER
self.conv_2 = tf.keras.layers.Conv3D(self.filter_size_2,(3,3,3), padding=self.padding, kernel_regularizer=self.l2 ,bias_regularizer=self.l1, trainable=trainable)
 self.batch_norm_2 = tf.keras.layers.BatchNormalization(trainable=trainable)
 self.relu_2 = tf.keras.layers.ReLU()
self.concat = tf.keras.layers.Concatenate(axis=-1)
 @tf.function
def call(self, x, training=False):
x_ = self.flatten(x)
 x = self.conv_0(x, training = training)
 x = self.batch_norm_0(x, training = training)
 x = self.relu_0(x, training = training)
 x = self.dropout(x, training = training)
 x = self.conv_1(x, training = training)
 x = self.batch_norm_1(x, training = training)
 x = self.relu_1(x, training = training)
 x = self.dropout(x, training = training)
 x = self.conv_2(x, training = training)
 x = self.batch_norm_2(x, training = training)
 x = self.relu_2(x, training = training)
 x = self.dropout(x, training = training)
 x = self.flatten(x)
 return self.concat([x, x_])
 class DenseBlock(tf.keras.layers.Layer):
def __init__(self, dense_size=512, x_is_flat=False, dropout=DENSE_DROPOUT):
super(DenseBlock, self ).__init__()
self.l2 = tf.keras.regularizers.L2(l2=DENSE_L1)
 self.l1 = tf.keras.regularizers.L1(l1=DENSE_L2)
 self.x_is_flat = x_is_flat
if not x_is_flat:
self.flatten = tf.keras.layers.Flatten()
self.concat = tf.keras.layers.Concatenate(axis=1)
 self.dense = tf.keras.layers.Dense(dense_size, kernel_regularizer=self.l2 ,bias_regularizer=self.l1)
 self.batch_norm = tf.keras.layers.BatchNormalization()
self.relu = tf.keras.layers.ReLU()
self.dropout = tf.keras.layers.Dropout(dropout)
 @tf.function
def call(self, x, other_x, training=False):
if not self.x_is_flat:
x_ = self.concat([self.flatten(x), tf.cast(other_x, 'float')])
 else:
x_ = self.concat([x, tf.cast(other_x, 'float')])
 x_ = self.dense(x_, training=training)
 x_ = self.batch_norm(x_,training=training)
 x_ = self.relu(x_, training=training)
 x_ = self.dropout(x_, training=training)
 return x_
cb_model = CatBoostClassifier(iterations=3000, learning_rate=0.02, depth=2, objective="Logloss" )<train_model>
class AutoCoordNet(tf.keras.Model):
def __init__(self, output_size=OUTPUT_SIZE, everything_trainable=True):
super(AutoCoordNet, self ).__init__()
self.auto_block = AutoCoordBlock(trainable=everything_trainable)
 self.res_block_0 = ResBlock(None, 1*CONV_FILTER, None, is_first=True, trainable=everything_trainable)
 self.res_block_1 = ResBlock(2*CONV_FILTER, 2*CONV_FILTER, 1*CONV_FILTER, trainable=everything_trainable)
 self.res_block_2 = ResBlock(4*CONV_FILTER, 4*CONV_FILTER, 2*CONV_FILTER, trainable=everything_trainable)
 self.reduce_block = ReduceBlock(trainable=everything_trainable)
 self.dense0 = DenseBlock(x_is_flat=True)
 self.dense1 = DenseBlock(dense_size=256, x_is_flat=True)
 self.output_layer = tf.keras.layers.Dense(output_size, activation='sigmoid')
 self.history = {'loss':[], 'val_loss':[] }
self.loss_metrics = tf.keras.metrics.BinaryCrossentropy()
def train(self, train_ds, validation_ds, epochs, reduce_lr, early_stopping):
self.onTrainBegin(reduce_lr, early_stopping)
 for epoch in range(1,epochs+1):
self.loss_metrics.reset_states()
start_time = time.time()
for step,(x_batch, other_x_batch, y_batch)in enumerate(train_ds):
self.train_step(x_batch, other_x_batch, y_batch)
 loss = self.loss_metrics.result().numpy()
self.history['loss'].append(loss)
 self.loss_metrics.reset_states()
for(val_x_batch, val_other_x_batch, val_y_batch)in validation_ds:
self.val_step(val_x_batch, val_other_x_batch, val_y_batch)
 val_loss = self.loss_metrics.result().numpy()
self.history['val_loss'].append(val_loss)
 print(f"Epoch {epoch}/{epochs} -","loss: %.4f,"%(float(loss)) ,"val_loss: %.4f,"%(float(val_loss)) , "time spent: %.2fs."%(time.time() -start_time))
self.onEpochEnd(epoch, val_loss)
 if self.stop_training == True:
return self.history
return self.history
def predict(self, ds):
y = []
for(x_batch, other_x_batch)in ds:
logits = self(x_batch, other_x_batch, training=False)
 y.extend(logits.numpy())
 return np.array(y)
 @tf.function
def train_step(self, x, other_x, y):
with tf.GradientTape() as tape:
y_pred = self(x, other_x, training=True)
 loss = self.compiled_loss(y, y_pred, regularization_losses=self.losses)
 gradients = tape.gradient(loss, self.trainable_variables)
 self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))
self.loss_metrics.update_state(y, y_pred)
 @tf.function
def val_step(self, x, other_x, y):
val_logits = self(x, other_x, training=False)
 self.loss_metrics.update_state(y, val_logits)
 @tf.function
def call(self, x, other_x, training=False):
x = self.auto_block(x, training=training)
 x = self.res_block_0(x, training=training)
 x = self.res_block_1(x, training=training)
 x = self.res_block_2(x, training=training)
 x = self.reduce_block(x, training=training)
 x = self.dense0(x, other_x, training=training)
 x = self.dense1(x, other_x, training=training)
 return self.output_layer(x, training=training)
 @tf.function
def onTrainBegin(self, reduce_lr, early_stopping):
self.reduce_lr, self.early_stopping = reduce_lr, early_stopping
self.reduce_lr.set_model(self), self.early_stopping.set_model(self)
 self.reduce_lr.on_train_begin() , self.early_stopping.on_train_begin()
def auto_train(self, trainable):
self.auto_block.trainable = trainable
def res_train(self, trainable):
self.res_block_0.trainable = trainable
self.res_block_1.trainable = trainable
self.res_block_2.trainable = trainable
self.reduce_block.trainable = trainable
def dense_train(self, trainable):
self.dense0.trainable = trainable
self.dense1.trainable = trainable
self.output_layer.trainable = trainable
def onEpochEnd(self, epoch, val_loss):
logs = {'val_loss': val_loss}
self.reduce_lr.on_epoch_end(epoch,logs), self.early_stopping.on_epoch_end(epoch,logs)
 def setOutputSize(self, size):
self.output_layer = tf.keras.layers.Dense(size, activation='sigmoid' )
cb_model.fit(X_train, y_train.ravel() , eval_set=[(X_valid, y_valid.ravel())],
early_stopping_rounds=1000 )<predict_on_test>
test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 test_genes = test_features.filter(regex=(GENES_REGEX)).to_numpy()
test_clean = cleanFeatures(test_features)
 X_test = rankGauss(test_genes)
 other_X_test = test_clean.filter(regex=(REGEX)).to_numpy()
test_ds = tf.data.Dataset.from_tensor_slices(( X_test, other_X_test)).batch(BATCH_SIZE)
pred_test_cb = cb_model.predict(test_df_new.values, prediction_type="Probability")[:, 1]
pred_test_cb[0:10]<save_to_csv>
def getCallbacks() :
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
mode='min',
factor=RLR_FACTOR,
patience=RLR_PATIENCE,
verbose=2)
 early_stopping = tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
mode='min',
patience=ES_PATIENCE,
restore_best_weights=True,
verbose=2)
 return reduce_lr, early_stopping
sub_df_cb = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_cb["target"] = pred_test_cb
sub_df_cb.to_csv("cb_submission.csv", index=False)
 sub_df_cb.head()<define_variables>
def getInfer() :
stratified_ds_generator = getMskfDataset(X, other_X, targets, all_targets)
 preds=[]
i = 0
for(X_train_0, other_X_train_0, y_train_0, X_val_0, other_X_val_0, y_val_0, X_train_1, other_X_train_1, y_train_1, X_val_1, other_X_val_1, y_val_1)in stratified_ds_generator:
i+=1
print(f'FOLD {i} starting...')
 train_ds_1 = tf.data.Dataset.from_tensor_slices(( X_train_1, other_X_train_1, y_train_1)).batch(BATCH_SIZE)
 val_ds_1 = tf.data.Dataset.from_tensor_slices(( X_val_1, other_X_val_1, y_val_1)).batch(BATCH_SIZE)
 model_1 = AutoCoordNet()
model_1.setOutputSize(OUTPUT_SIZE)
 model_1.compile(loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),
optimizer = tf.keras.optimizers.SGD(learning_rate=LEARNING_RATE, momentum=.9))
rlr, es = getCallbacks()
model_1.train(train_ds_1, val_ds_1, epochs=EPOCHS, reduce_lr=rlr, early_stopping=es)
 plotHistory(model_1)
 pred = model_1.predict(test_ds)
 preds.append(pred)
 infer = np.zeros(( test_features.shape[0],206))
for pred_ in preds:
infer += pred_
infer /= N_SPLITS
return infer
pred_ensemble = 0.4 * pred_test_lgb + 0.3 * pred_test_xgb + 0.3 * pred_test_cb
pred_ensemble[0:10]<save_to_csv>
infer = getInfer()
ctl_idx = np.where(test_features.iloc[:,1].to_numpy() == 'ctl_vehicle')
 infer[ctl_idx] = 0
sub_df_ensemble = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_ensemble["target"] = pred_ensemble
sub_df_ensemble.to_csv("ensemble_submission.csv", index=False)
 sub_df_ensemble.head()<load_from_csv>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = infer
sub.to_csv('submission.csv', index=False )