Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<load_from_csv>
PROBE_COLUMN = 0
sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 print('Submission file has shape', sub.shape)
 print('We will probe target',sub.columns[1+PROBE_COLUMN] )
def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index'])
 return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 16, 3, 12)
 x = BatchNormalization()(x)
 x = wave_block(x, 32, 3, 8)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 3:
lr = LR
elif epoch < 10:
lr = LR / 3
elif epoch < 30:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 model.save("model-wavenet.h5")
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f' )<split>
sub.iloc[:,1:] = 0
sub.iloc[:,1+PROBE_COLUMN] = 1
sub.to_csv('submission_probe_%i.csv'%PROBE_COLUMN)
 sub.head()
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...' )<train_on_grid>
np.round(( 0.29790 - 0.13073)/(-np.log(1e-15)/820292))
print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...' )<import_modules>
test_counts = pd.read_csv('.. /input/moa-test-target-means/target_counts.csv')
 test_counts.head()
import numpy as np
import pandas as pd
from catboost import cv, Pool<load_from_csv>
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' )
train = pd.read_csv('.. /input/data-without-drift/train_clean.csv', dtype={'time':np.float32,'signal':np.float32,'open_channels':np.int32})
 test = pd.read_csv('.. /input/data-without-drift/test_clean.csv', dtype={'time':np.float32,'signal':np.float32})
 sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv', dtype={'time':np.float32,'open_channels':np.int32})
 Y_train_proba = np.load(".. /input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load(".. /input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:,i]
test[f"proba_{i}"] = Y_test_proba[:,i]
train['proba'] = np.argmax(Y_train_proba,axis=1)
 test['proba'] = np.argmax(Y_test_proba,axis=1 )<prepare_x_and_y>
test_counts.loc[test_counts.train_ct==6].target.iloc[0]
train.loc[train.index % 2 ==0,'id'] = 1
train.loc[train.index % 2 !=0,'id'] = 2
x_train = train[train.id ==1].drop(['open_channels','id'],axis=1)
 y_train = train[train.id ==1]['open_channels']
x_val = train[train.id ==2].drop(['open_channels','id'],axis=1)
 y_val = train[train.id ==2]['open_channels']
print(x_train.shape)
 print(y_train.shape)
 print(x_val.shape)
 print(y_val.shape )<train_model>
log_loss(train_targets['diuretic'], np.ones(23814)*6/23814 )
cat = CatBoostClassifier(task_type='GPU',iterations=10000,loss_function='MultiClass',od_type='Iter',od_wait=500)
 cat.fit(x_train,y_train,verbose=25, eval_set =(x_val,y_val),cat_features=[13])
<save_to_csv>
test_counts.loc[test_counts.train_ct==36].target.iloc[0]
y_preds = cat.predict(test)
 sub['open_channels'] = y_preds
sub.to_csv('submission_cat.csv',index=False,float_format='%.4f' )<define_variables>
log_loss(train_targets['casein_kinase_inhibitor'], np.ones(23814)*36/23814 )
EPOCHS = 180
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.0015
SPLITS = 6
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<load_from_csv>
( 0.01132 - 0.00234)* 60 / 206
def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features<categorify>
! python -W ignore.. /input/finalmoascripts/inference-tabnet-1830.py
sub = pd.read_csv('submission_tabnet_01830.csv')
 test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv' )
train, test, sample_submission = read_data()
train, test = normalize(train, test )<feature_engineering>
def scale(x,k):
x = x.copy()
idx = np.where(x!=1)[0]
y = k * x[idx] /(1-x[idx])
 x[idx] = y/(1+y)
 return x
train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test )<feature_engineering>
public = pd.read_csv('.. /input/moa-test-target-means/sample_public_submission.csv' ).sig_id.values
SZ = 3982; SZ2 = 3982*3
for c in test_counts.target.values:
m2 = sub.loc[sub.sig_id.isin(public),c].mean()
m3 = sub.loc[~sub.sig_id.isin(public),c].mean()
m = test_counts.loc[test_counts.target==c,'public_ct'].values[0]/SZ /m2
sub.loc[sub.sig_id.isin(public),c] = scale(sub.loc[sub.sig_id.isin(public),c].values,m)
 if len(sub)>len(public):
m = test_counts.loc[test_counts.target==c,'private_ct'].values[0]/SZ2 /m3
sub.loc[~sub.sig_id.isin(public),c] = scale(sub.loc[~sub.sig_id.isin(public),c].values,m)
 print('Updated',len(test_counts),'targets' )
<import_modules><EOS> 
s = sub.loc[(test.cp_type=='ctl_vehicle' ).values].mean().sum()
print('Control rows in submission have mean = %i'%s)
 sub.to_csv('submission.csv',index=False)
 sub.head()