Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<compute_test_metric>
warnings.filterwarnings('ignore' )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<choose_model_class>
from sklearn.preprocessing import QuantileTransformer
model = Simple_NN(208,512)
 model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr = 0.0002 )<define_variables>
import os
import gc
import pickle
import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
from tqdm.notebook import tqdm
from time import time
n_epochs = 40
batch_size = 512
train_preds = np.zeros(( len(train_features)))
 test_preds = np.zeros(( len(test_features)))
 x_test = np.array(test_features)
 x_test_cuda = torch.tensor(x_test, dtype=torch.float ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(train_features)
 y_train = np.array(train_target)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.float ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.float ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
loss_fn = torch.nn.BCEWithLogitsLoss()
step_size = 2000
base_lr, max_lr = 0.0001, 0.005
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch)in enumerate(train_loader):
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() /len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_features)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 auc_nn = round(roc_auc_score(train_target,train_preds),4)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t auc={:.4f}'.format(np.average(avg_losses_f),np.average(avg_val_losses_f),auc_nn))<define_variables>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
from tensorflow.keras import layers,regularizers,Sequential,backend,callbacks,optimizers,metrics,losses
ensemble = 0.5*train_preds_lgb + 0.5* train_preds
ensemble_test = 0.5* y_pred_lgb + 0.5* test_preds<compute_test_metric>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
print('LightBGM auc = {:<8.5f}'.format(auc_lgb))
print('NN auc = {:<8.5f}'.format(auc_nn))
print('NN+LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, ensemble)) )<save_to_csv>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
submission_lgb = pd.DataFrame({
"ID_code": test_ID,
"target": y_pred_lgb
})
 submission_lgb.to_csv('submission_lgb.csv', index=False )<save_to_csv>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
submission_nn = pd.DataFrame({
"ID_code": test_ID,
"target": test_preds
})
 submission_nn.to_csv('submission_nn.csv', index=False )<save_to_csv>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
submission_ens = pd.DataFrame({
"ID_code": test_ID,
"target": ensemble_test
})
 submission_ens.to_csv('submission_ens.csv', index=False )<save_to_csv>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
<set_options>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
warnings.filterwarnings('ignore' )<load_from_csv>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<feature_engineering>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
i = 1
for df in [test, train]:
idx = df.columns.values[i:i+200]
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 i = i + 1<categorify>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<prepare_x_and_y>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
<define_variables>
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
ss_rf = ss_svc.copy()
ss = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']
features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']<init_hyperparams>
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 28,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1}<train_model>
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=31415)
 oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 15000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
def create_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(4096, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(4096, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(4096, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.001),metrics=logloss 
)
 return model
sub_df = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )<prepare_x_and_y>
top_feats = range(1039)
 print(len(top_feats))
model = CatBoostClassifier(loss_function="Logloss", eval_metric="AUC")
 kf = KFold(n_splits=10, random_state=42, shuffle=True)
 y_valid_pred = 0 * target
y_test_pred = 0
for idx,(train_index, valid_index)in enumerate(kf.split(train)) :
y_train, y_valid = target.iloc[train_index], target.iloc[valid_index]
X_train, X_valid = train[features].iloc[train_index,:], train[features].iloc[valid_index,:]
_train = Pool(X_train, label=y_train)
 _valid = Pool(X_valid, label=y_valid)
 print("
Fold ", idx)
 fit_model = model.fit(_train,
eval_set=_valid,
use_best_model=True,
verbose=200 
)
 pred = fit_model.predict_proba(X_valid)[:,1]
print(" auc = ", roc_auc_score(y_valid, pred))
y_valid_pred.iloc[valid_index] = pred
y_test_pred += fit_model.predict_proba(test[features])[:,1]
y_test_pred /= 5<save_to_csv>
scaler = StandardScaler()
X = scaler.fit_transform(train.values)
 x_tt = scaler.transform(test_features.values )
sub_df1 = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub_df1["target"] = y_test_pred
sub_df1.to_csv("cat_submission.csv", index=False )<save_to_csv>
train = train.astype('float')
 test = test.astype('float' )
sub_df2 = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub_df2["target"] = 0.5*sub_df["target"] + 0.5*sub_df1["target"]
sub_df2.to_csv("lgb_cat_submission.csv", index=False )<load_from_csv>
p_min = 0.001
p_max = 0.999
df = pd.read_csv(".. /input/train.csv")
 df.head()<count_missing_values>
N_STARTS = 1
tf.random.set_seed(42)
 ss_NN = ss_svc.copy()
res_NN = train_targets.copy()
ss_NN.loc[:, train_targets.columns] = 0
res_NN.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=12, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f'Fold {n}')
 model = create_model(len(top_feats))
checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min')
 model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=35, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt], verbose=2, 
)
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss_NN.loc[:, train_targets.columns] += test_predict
res_NN.loc[te, train_targets.columns] += val_predict
print('')
 ss_NN.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res_NN.loc[:, train_targets.columns] /= N_STARTS
df.isnull().any().any()<count_values>
ss_NN.loc[:, train_targets.columns] = np.clip(ss_NN[train_targets.columns].values,p_min,p_max)
 res_NN.loc[:, train_targets.columns] = np.clip(res_NN[train_targets.columns].values,p_min,p_max)
 print(f'NN OOF Metric: {log_loss_metric(train_targets, res_NN)}')
 res_NN.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_NN.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )
df['target'].value_counts(normalize=True )<drop_column>
ss_NN.to_csv('submission.csv', index = False )