Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<prepare_x_and_y>
import numpy as np
import pandas as pd
import os
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import time
n_epochs = 40
batch_size = 256
train_preds = np.zeros(( len(train_features)))
 test_preds = np.zeros(( len(test_features)))
 x_test = np.array(test_features)
 x_test_cuda = torch.tensor(x_test, dtype=torch.float ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(train_features)
 y_train = np.array(train_target)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.float ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.float ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
loss_fn = torch.nn.BCEWithLogitsLoss()
model = Simple_NN(200,16)
 model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr = 0.001,weight_decay=1e-5)
 step_size = 2000
base_lr, max_lr = 0.001, 0.005
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch)in enumerate(train_loader):
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() /len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_features)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 auc = round(roc_auc_score(train_target,train_preds),4)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t auc={:.4f}'.format(np.average(avg_losses_f),np.average(avg_val_losses_f),auc))<drop_column>
test_df = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_df = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_target_df = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 target_cols = train_target_df.columns[1:]
N_TARGETS = len(target_cols)
 print(train_df.shape )
train_features = train_df.drop(['target','ID_code'], axis = 1)
 test_features = test_df.drop(['ID_code'],axis = 1)
 train_target = train_df['target']<init_hyperparams>
GENES = [col for col in train_df.columns if col.startswith('g-')]
CELLS = [col for col in train_df.columns if col.startswith('c-')]
param = {
'num_leaves': 7,
'learning_rate': 0.01,
'feature_fraction': 0.04,
'max_depth': 17,
'objective': 'binary',
'boosting_type': 'gbdt',
'metric': 'auc',
}<prepare_x_and_y>
n_comp = 50
data = pd.concat([pd.DataFrame(train_df[GENES]), pd.DataFrame(test_df[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_df.shape[0]]; test2 = data2[-test_df.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_df = pd.concat(( train_df, train2), axis=1)
 test_df = pd.concat(( test_df, test2), axis=1 )
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features)
 y_train = np.array(train_target)
 trn_data = lgb.Dataset(x_train[train_idx.astype(int)], label=y_train[train_idx.astype(int)])
 val_data = lgb.Dataset(x_train[valid_idx.astype(int)], label=y_train[valid_idx.astype(int)])
 num_round = 15000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)
 oof[valid_idx] = clf.predict(x_train[valid_idx], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = i + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_features, num_iteration=clf.best_iteration)/ 5
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof)) )<compute_test_metric>
n_comp = 15
data = pd.concat([pd.DataFrame(train_df[CELLS]), pd.DataFrame(test_df[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_df.shape[0]]; test2 = data2[-test_df.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_df = pd.concat(( train_df, train2), axis=1)
 test_df = pd.concat(( test_df, test2), axis=1 )
esemble = 0.6*oof + 0.4* train_preds
print('NN auc = {:<8.5f}'.format(auc))
print('LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, oof)))
 print('NN+LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, esemble)) )<define_variables>
train_copy = train_df
var_thresh = VarianceThreshold(0.8)
 data = train_df.append(test_df)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_df_transformed = data_transformed[ : train_df.shape[0]]
test_df_transformed = data_transformed[-test_df.shape[0] : ]
train_df = pd.DataFrame(train_df[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_df = pd.concat([train_df, pd.DataFrame(train_df_transformed)], axis=1)
 test_df = pd.DataFrame(test_df[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_df = pd.concat([test_df, pd.DataFrame(test_df_transformed)], axis=1)
 train_df.shape
esemble_pred = 0.4* test_preds+ 0.6 *predictions<define_variables>
search_row = dict(train_copy.iloc[0,4:])
 col_rela = {}
for i in np.arange(0,868):
for k, v in search_row.items() :
if train_df[i][0] == v:
col_rela[i] = k
train_df = train_df.rename(columns=col_rela)
 test_df = test_df.rename(columns=col_rela )
id_code_test = test_df['ID_code']<create_dataframe>
SEED = 1234
EPOCHS = 28
BATCH_SIZE = 128
FOLDS = 5
REPEATS = 5
LR = 0.0005
N_TARGETS = len(target_cols )
my_submission_nn = pd.DataFrame({"ID_code" : id_code_test, "target" : test_preds})
 my_submission_lbgm = pd.DataFrame({"ID_code" : id_code_test, "target" : predictions})
 my_submission_esemble = pd.DataFrame({"ID_code" : id_code_test, "target" : esemble_pred} )<save_to_csv>
def seed_everything(seed):
np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
my_submission_nn.to_csv('submission_nn.csv', index = False, header = True)
 my_submission_lbgm.to_csv('submission_lbgm.csv', index = False, header = True)
 my_submission_esemble.to_csv('submission_esemble.csv', index = False, header = True )<import_modules>
def multi_log_loss(y_true, y_pred):
losses = []
for col in y_true.columns:
losses.append(log_loss(y_true.loc[:, col], y_pred.loc[:, col]))
return np.mean(losses )
tqdm.pandas() 
<set_options>
def preprocess_df(data):
data['cp_type'] =(data['cp_type'] == 'trt_cp' ).astype(int)
 data['cp_dose'] =(data['cp_dose'] == 'D2' ).astype(int)
 return data
%matplotlib inline
%precision 4
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 np.set_printoptions(suppress=True)
 pd.set_option("display.precision", 15 )<load_from_csv>
x_train = preprocess_df(train_df.drop(columns="sig_id"))
x_test =preprocess_df(test_df.drop(columns="sig_id"))
y_train = train_target_df.drop(columns="sig_id")
 N_FEATURES = x_train.shape[1]
train= pd.read_csv(".. /input/train.csv")
 test = pd.read_csv('.. /input/test.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv' )<define_variables>
x_train = x_train.astype({'cp_time':int})
 x_test = x_test.astype({'cp_time':int} )
<count_missing_values>
def create_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(N_TARGETS, activation="sigmoid"))
])
 model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = LR), loss='binary_crossentropy', metrics=["accuracy"])
 return model
def check_missing_data(df):
flag=df.isna().sum().any()
if flag==True:
total = df.isnull().sum()
percent =(df.isnull().sum())/(df.isnull().count() *100)
 output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 data_type = []
for col in df.columns:
dtype = str(df[col].dtype)
 data_type.append(dtype)
 output['Types'] = data_type
return(np.transpose(output))
else:
return(False )<count_missing_values>
def build_train(resume_models = None, repeat_number = 0, folds = 5, skip_folds = 0):
models = []
oof_preds = y_train.copy()
kfold = KFold(folds, shuffle = True)
 for fold,(train_ind, val_ind)in enumerate(kfold.split(x_train)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'binary_crossentropy', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'auto')
 checkpoint_path = f'repeat:{repeat_number}_Fold:{fold}.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 model = create_model()
model.fit(x_train.values[train_ind],
y_train.values[train_ind],
validation_data=(x_train.values[val_ind], y_train.values[val_ind]),
callbacks = [cb_lr_schedule, cb_checkpt],
epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=2 
)
 model.load_weights(checkpoint_path)
 oof_preds.loc[val_ind, :] = model.predict(x_train.values[val_ind])
 models.append(model)
 print('train:')
 print(list(zip(model.metrics_names, model.evaluate(x_train.values[train_ind], y_train.values[train_ind], verbose=0, batch_size=32))))
print('val:')
 print(list(zip(model.metrics_names, model.evaluate(x_train.values[val_ind], y_train.values[val_ind], verbose=0, batch_size=32))))
return models, oof_preds
print('missing in train: ',check_missing_data(train))
print('missing in test: ',check_missing_data(test))<categorify>
start = time.time()
models = []
oof_preds = []
seed_everything(SEED)
 for i in range(REPEATS):
m, oof = build_train(repeat_number = i, folds=FOLDS)
 models = models + m
oof_preds.append(oof)
 finish = time.time() -start
print(finish )
<feature_engineering>
models[1].predict(x_test )
<prepare_x_and_y><EOS> 
test_preds = sub.copy()
test_preds[target_cols] = 0
for model in models:
test_preds.loc[:,target_cols] += model.predict(x_test)
 test_preds.loc[:,target_cols] /= len(models)
 test_preds.loc[x_test['cp_type'] == 0, target_cols] = 0
test_preds.to_csv('submission.csv', index=False )