Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
import numpy as np
import pandas as pd
import os
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
n = train.shape[0]
vec = TfidfVectorizer(tokenizer=tokenize,
min_df=2,
max_df=.625,
strip_accents='unicode',
use_idf=1,
smooth_idf=True,
sublinear_tf=True)
 trn_term_doc = vec.fit_transform(train[COMMENT])
 test_term_doc = vec.transform(test[COMMENT] )<compute_test_metric>
test_df = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_df = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_target_df = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 target_cols = train_target_df.columns[1:]
N_TARGETS = len(target_cols )
def pr(y_i, y):
p = x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1 )<prepare_x_and_y>
SEED = 1234
EPOCHS = 28
BATCH_SIZE = 128
FOLDS = 5
REPEATS = 5
LR = 0.0005
N_TARGETS = len(target_cols )
x = trn_term_doc
test_x = test_term_doc<train_on_grid>
def seed_everything(seed):
np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
class random_log_uniform:
def __init__(self, power_low, power_high):
self.power_low = power_low
self.power_high = power_high
def rvs(self, random_state=0):
rnd_power = np.random.rand() *(self.power_high - self.power_low)+ self.power_low
return 10**rnd_power
def get_mdl_training(y, parameters):
y = y.values
r = np.log(pr(1,y)/ pr(0,y))
model = LogisticRegression(dual=True)
 params = { "C": random_log_uniform(np.log10(0.01), np.log10(10)) }
x_nb = x.multiply(r)
 model_grid = RandomizedSearchCV(model, params, cv=5, n_jobs=1, n_iter=50, scoring='roc_auc')
 model_grid.fit(x_nb,y)
 m = model_grid.best_estimator_
parameters.append(model_grid.best_params_["C"])
 return m.fit(x_nb, y), r
def get_mdl(y,C):
y = y.values
r = np.log(pr(1,y)/ pr(0,y))
m = LogisticRegression(C=C, dual=True)
 x_nb = x.multiply(r)
 return m.fit(x_nb, y), r<train_model>
def multi_log_loss(y_true, y_pred):
losses = []
for col in y_true.columns:
losses.append(log_loss(y_true.loc[:, col], y_pred.loc[:, col]))
return np.mean(losses )
preds = np.zeros(( len(test), len(label_cols)))
 Cvalues = [1.391737250242211, 0.17361472592588692, 0.45345054479480851,
0.58227213530514976, 0.59166916356898391, 0.34278082413675537]
TRAINING_HYPERPARAMETERS = False
if TRAINING_HYPERPARAMETERS:
Cvalues = []
for i, j in enumerate(label_cols):
print('fitting %s.. ' %j)
 if TRAINING_HYPERPARAMETERS:
m,r = get_mdl_training(train[j], Cvalues)
 else:
m,r = get_mdl(train[j], Cvalues[i])
 preds[:,i] = m.predict_proba(test_x.multiply(r)) [:,1]
if TRAINING_HYPERPARAMETERS:
print("\tfitted coefficients: ", Cvalues )<save_to_csv>
def preprocess_df(data):
data['cp_type'] =(data['cp_type'] == 'trt_cp' ).astype(int)
 data['cp_dose'] =(data['cp_dose'] == 'D2' ).astype(int)
 return data
submid = pd.DataFrame({'id': subm["id"]})
 submission = pd.concat([submid, pd.DataFrame(preds, columns = label_cols)], axis=1)
 submission.to_csv('submission.csv', index=False )<compute_test_metric>
x_train = preprocess_df(train_df.drop(columns="sig_id"))
x_test =preprocess_df(test_df.drop(columns="sig_id"))
y_train = train_target_df.drop(columns="sig_id")
 N_FEATURES = x_train.shape[1]
if SPLIT_TRAINING:
for index, _ in enumerate(label_cols):
print("%.5f" % roc_auc_score(test[label_cols].values[:,index], preds[:,index]), end=" ")
 print("
%.5f" %roc_auc_score(test[label_cols].values, preds))<import_modules>
def create_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(N_TARGETS, activation="sigmoid"))
])
 model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = LR), loss='binary_crossentropy', metrics=["accuracy"])
 return model
import pandas as pd
import numpy as np<import_modules>
def build_train(resume_models = None, repeat_number = 0, folds = 5, skip_folds = 0):
models = []
oof_preds = y_train.copy()
kfold = KFold(folds, shuffle = True)
 for fold,(train_ind, val_ind)in enumerate(kfold.split(x_train)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'binary_crossentropy', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'auto')
 checkpoint_path = f'repeat:{repeat_number}_Fold:{fold}.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 model = create_model()
model.fit(x_train.values[train_ind],
y_train.values[train_ind],
validation_data=(x_train.values[val_ind], y_train.values[val_ind]),
callbacks = [cb_lr_schedule, cb_checkpt],
epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=2 
)
 model.load_weights(checkpoint_path)
 oof_preds.loc[val_ind, :] = model.predict(x_train.values[val_ind])
 models.append(model)
 return models, oof_preds
import pandas as pd
import numpy as np<set_options>
models = []
oof_preds = []
seed_everything(SEED)
 for i in range(REPEATS):
m, oof = build_train(repeat_number = i, folds=FOLDS)
 models = models + m
oof_preds.append(oof)
warnings.filterwarnings('ignore')
<define_variables>
mean_oof_preds = y_train.copy()
mean_oof_preds.loc[:, target_cols] = 0
for i, p in enumerate(oof_preds):
print(f"Repeat {i + 1} OOF Log Loss: {multi_log_loss(y_train, p)}")
 mean_oof_preds.loc[:, target_cols] += p[target_cols]
mean_oof_preds.loc[:, target_cols] /= len(oof_preds)
 print(f"Mean OOF Log Loss: {multi_log_loss(y_train, mean_oof_preds)}")
 mean_oof_preds.loc[x_train['cp_type'] == 0, target_cols] = 0
print(f"Mean OOF Log Loss(ctl adjusted): {multi_log_loss(y_train, mean_oof_preds)}" )
<load_from_csv><EOS> 
test_preds = sub.copy()
test_preds[target_cols] = 0
for model in models:
test_preds.loc[:,target_cols] += model.predict(x_test)
 test_preds.loc[:,target_cols] /= len(models)
 test_preds.loc[x_test['cp_type'] == 0, target_cols] = 0
test_preds.to_csv('submission.csv', index=False )