Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
import os
import random
import time
import json
import warnings
import numpy as np
import pandas as pd
from sklearn.utils.class_weight import compute_class_weight
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.base import BaseEstimator
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, QuantileTransformer
from sklearn.metrics import fbeta_score
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import regularizers
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
import matplotlib.pyplot as plt
from xgboost import XGBClassifier
train['question_text'] = train['question_text'].apply(lambda x: x.replace('Quorans', 'Quora'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('Brexit', 'Europe'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('₹', 'rupee'))<feature_engineering>
sys.path += ['/kaggle/input/iterstrat', '/kaggle/input/tabnet']
vocab = build_vocab(train['question_text'])
 kw, uw = check_coverage(vocab, embed_index )<feature_engineering>
def set_random_seeds(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 set_random_seeds(43 )
train['question_text'] = train['question_text'].apply(lambda x: x.replace("'", ''))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('“', ''))<feature_engineering>
X_full = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 Y_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 Y_nonscored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 drug = pd.read_csv('/kaggle/input/lish-moa/train_drug.csv')
 X_test_full = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sig_id = X_test_full['sig_id'].values.reshape(( -1, 1))
vocab = build_vocab(train['question_text'])
 kw, uw = check_coverage(vocab, embed_index )<feature_engineering>
cat_cols = ['cp_type', 'cp_time', 'cp_dose']
numerical_cols = [c for c in X_full.columns if c not in ['sig_id'] + cat_cols]
label_cols = [c for c in Y_scored.columns if c != 'sig_id']
num_features = X_full.shape[1] - 1
num_labels = Y_scored.shape[1] - 1
num_nonscored_labels = Y_nonscored.shape[1] - 1
train['question_text'] = train['question_text'].apply(lambda x: x.replace('cryptocurrencies', 'cryptocurrency'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('Redmi', 'mobile company'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('OnePlus', 'mobile company'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('Qoura', 'Quora'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('°C', 'degree celsius'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('etc…', 'etc.'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('Bhakts', 'supporter'))
train['question_text'] = train['question_text'].apply(lambda x: x.replace('bhakts', 'supporter'))<feature_engineering>
def logloss(Y_true, Y_preds, label_smoothing=0):
Y_true = Y_true.astype(np.float32)
 return tf.reduce_mean(keras.losses.binary_crossentropy(Y_true, Y_preds, label_smoothing=label_smoothing)).numpy()
vocab = build_vocab(train['question_text'])
 kw, uw = check_coverage(vocab, embed_index )<feature_engineering>
def add_outputs(Y, Y_preds, X=None, idx_list=None, col=None):
if X is not None:
if col is None:
Y.loc[X['cp_type'] == 'trt_cp', :] += Y_preds
else:
Y.loc[X['cp_type'] == 'trt_cp', Y.columns[col]] += Y_preds
elif idx_list is not None:
if col is None:
Y.iloc[idx_list, :] += Y_preds
else:
Y.iloc[idx_list, col] += Y_preds
else:
raise Exception("Must provide X or idx_list" )
test['question_text'] = test['question_text'].apply(lambda x : x.replace("/"," "))
test['question_text'] = test['question_text'].apply(lambda x : x.replace("-"," "))
test['question_text'] = test['question_text'].apply(lambda x : clean_numbers(x))
test['question_text'] = test['question_text'].apply(lambda x : clean_punctuations(x))
test['question_text'] = test['question_text'].apply(lambda x : clean_apostrophe(x))
test['question_text'] = test['question_text'].apply(lambda x : remove_stopwords(x))
test['question_text'] = test['question_text'].apply(lambda x : re.sub("'s", ' is', x))
test['question_text'] = test['question_text'].apply(lambda x: x.replace("'", ''))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('“', ''))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('Quorans', 'Quora'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('Brexit', 'Europe'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('₹', 'rupee'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('cryptocurrencies', 'cryptocurrency'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('Redmi', 'mobile company'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('OnePlus', 'mobile company'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('Qoura', 'Quora'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('°C', 'degree celsius'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('etc…', 'etc.'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('Bhakts', 'supporter'))
test['question_text'] = test['question_text'].apply(lambda x: x.replace('bhakts', 'supporter')) 
<load_from_csv>
def balance_class_weights(Y_true, class_weights=(0.7, 1.6)) :
n_labels = Y_true.shape[1]
weights = np.empty([n_labels, 2])
 for i in range(n_labels):
if class_weights == 'balanced':
total = len(Y_true[:, i])
 pos = np.sum(Y_true[:, i])
 neg = total - pos
weight_for_0 =(1 / neg)*(total)/2.0
weight_for_1 =(1 / pos)*(total)/2.0
weights[i] = [weight_for_0, weight_for_1]
else:
weights[i] = np.array(class_weights)
 return weights
def weighted_binary_crossentropy(weights, label_smoothing=0):
def weighted_loss(Y_true, Y_pred):
bce = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 return K.mean(
(weights[:, 0]**(1 - Y_true)) *(weights[:, 1]**(Y_true)) * bce(Y_true, Y_pred),
axis=-1 
)
 return weighted_loss
X_train = train["question_text"].values
y = pd.read_csv('.. /input/train.csv')['target']
X_test = test["question_text"].values
x_train, x_val, y_train, y_val = train_test_split(X_train, y , test_size = 0.2 )<compute_test_metric>
def preprocess(df, targets=None):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df = df.drop(columns='sig_id')
 if targets is not None:
targets = targets.copy()
targets = targets[df['cp_type'] == 0].reset_index(drop=True)
 df = df[df['cp_type'] == 0].reset_index(drop=True)
 if targets is None:
return df
return df, targets
emb_mean, emb_std = all_embed.mean() , all_embed.std()<string_transform>
X_train, Y_train = preprocess(X_full, Y_scored)
 X_test = preprocess(X_test_full)
 Y_train_nonscored = Y_nonscored.drop(columns='sig_id')[X_full['cp_type'] == 'trt_cp'].reset_index(drop=True)
 initial_bias = -np.log(Y_train.mean(axis=0 ).values)
 nonscored_initial_bias = -np.log(Y_train_nonscored.mean(axis=0 ).values + 1e-6)
 Y_train_with_drugs = Y_train.merge(drug, on='sig_id', how='left')
 Y_train = Y_train.drop(columns='sig_id')
 vc = Y_train_with_drugs.drug_id.value_counts()
vc1 = vc.loc[(vc == 6)|(vc == 12)|(vc == 18)].index.sort_values()
vc2 = vc.loc[(vc != 6)&(vc != 12)&(vc != 18)].index.sort_values()
tokenizer = Tokenizer(num_words = 50000)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_val = tokenizer.texts_to_sequences(x_val)
 x_test = tokenizer.texts_to_sequences(X_test )<drop_column>
def select_features(X):
return X.iloc[:, [
1, 2, 3, 4, 5, 6, 7, 9, 11, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31,
32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46,
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 58, 59, 60,
61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88,
89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101,
102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 120, 121, 122, 123, 124, 125, 126, 127, 128,
129, 130, 131, 132, 133, 136, 137, 138, 139, 140, 141, 142, 143,
144, 145, 146, 147, 149, 150, 151, 152, 153, 154, 155, 156, 157,
158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170,
171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183,
184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 197,
198, 199, 200, 202, 203, 204, 205, 206, 208, 209, 210, 211, 212,
213, 214, 215, 216, 217, 218, 219, 220, 221, 223, 224, 225, 226,
227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253,
254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266,
267, 268, 269, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280,
281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294,
295, 296, 298, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309,
310, 311, 312, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336,
337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 367, 368, 369, 370, 371, 374, 375, 376, 377,
378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 390, 391,
392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404,
405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 416, 417, 418,
419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431,
432, 434, 435, 436, 437, 438, 439, 440, 442, 443, 444, 445, 446,
447, 448, 449, 450, 453, 454, 456, 457, 458, 459, 460, 461, 462,
463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475,
476, 477, 478, 479, 481, 482, 483, 484, 485, 486, 487, 488, 489,
490, 491, 492, 493, 494, 495, 496, 498, 500, 501, 502, 503, 505,
506, 507, 509, 510, 511, 512, 513, 514, 515, 518, 519, 520, 521,
522, 523, 524, 525, 526, 527, 528, 530, 531, 532, 534, 535, 536,
538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 549, 550, 551,
552, 554, 557, 559, 560, 561, 562, 565, 566, 567, 568, 569, 570,
571, 572, 573, 574, 575, 577, 578, 580, 581, 582, 583, 584, 585,
586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 599,
600, 601, 602, 606, 607, 608, 609, 611, 612, 613, 615, 616, 617,
618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630,
631, 632, 633, 634, 635, 636, 637, 638, 639, 641, 642, 643, 644,
645, 646, 647, 648, 649, 650, 651, 652, 654, 655, 656, 658, 659,
660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672,
673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685,
686, 687, 688, 689, 691, 692, 693, 694, 695, 696, 697, 699, 700,
701, 702, 704, 705, 707, 708, 709, 710, 711, 713, 714, 716, 717,
718, 720, 721, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732,
733, 734, 735, 737, 738, 739, 740, 742, 743, 744, 745, 746, 747,
748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 759, 760, 761,
762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774,
775, 776, 777, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788,
789, 790, 792, 793, 794, 795, 796, 797, 798, 800, 801, 802, 803,
804, 805, 806, 808, 809, 811, 813, 814, 815, 816, 817, 818, 819,
821, 822, 823, 825, 826, 827, 828, 829, 830, 831, 832, 834, 835,
837, 838, 839, 840, 841, 842, 845, 846, 847, 848, 850, 851, 852,
854, 855, 856, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869,
870, 871, 872, 873, 874
]].values
del train, test<drop_column>
gene_cols = [c for c in X_test.columns if c.startswith('g-')]
cell_cols = [c for c in X_test.columns if c.startswith('c-')]
del X_train, X_test<string_transform>
qtransform = joblib.load('/kaggle/input/qtransform/qtransform.joblib')
 X_test_selected = select_features(X_test)
 X_test_qtrans = np.concatenate([X_test_selected[:, :2], qtransform.transform(X_test_selected[:, 2:])], axis=1)
 pca_qtrans_2 = joblib.load('/kaggle/input/qtransform/pca_qtrans.joblib' )
maxlen = 100
x_train = pad_sequences(x_train, maxlen = maxlen)
 x_val = pad_sequences(x_val, maxlen = maxlen)
 x_test = pad_sequences(x_test, maxlen = maxlen )<define_variables>
std_scaler_2 = joblib.load('/kaggle/input/qtransform/std_scaler_2.joblib')
 gene_pca_2 = joblib.load('/kaggle/input/qtransform/gene_pca_2.joblib')
 cell_pca_2 = joblib.load('/kaggle/input/qtransform/cell_pca_2.joblib' )
word_index = tokenizer.word_index
nb_words = min(50000, len(word_index))
print(nb_words )<compute_test_metric>
kmeans_g = joblib.load('/kaggle/input/qtransform/kmeans_g.joblib')
 kmeans_c = joblib.load('/kaggle/input/qtransform/kmeans_c.joblib' )
embed_matrix = np.random.normal(emb_mean, emb_std,(nb_words, 300))<feature_engineering>
def preprocess_nn(X):
X_pca = pca_qtrans_2.transform(X)
 return np.concatenate([X, X_pca], axis=1 )
for word, i in word_index.items() :
if i >= 50000: continue
embed_vector = embed_index.get(word)
 if embed_vector is not None:
embed_matrix[i] = embed_vector<choose_model_class>
X_test_nn = preprocess_nn(X_test_qtrans )
def get_model() :
inp = Input(shape =(maxlen,))
x = Embedding(50000, 300, weights = [embed_matrix] )(inp)
 x = Bidirectional(LSTM(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation='relu' )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 return model<train_model>
class MultiLabelNN(BaseEstimator):
def __init__(self,
n_input_features=num_features,
n_output_labels=num_labels,
n_hidden_units=2048):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.n_hidden_units = n_hidden_units
self._model_nn = None
def _create_nn_model(self, loss_fn=None, loss_fn_aux=None):
output_bias = keras.initializers.Constant(initial_bias)
 aux_output_bias = keras.initializers.Constant(nonscored_initial_bias)
 inp = keras.layers.Input(self.n_input_features)
 gauss_noise = keras.layers.GaussianNoise(0.001 )(inp)
 norm_1 = keras.layers.BatchNormalization()(gauss_noise)
 dense_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(norm_1)
 norm_2 = keras.layers.BatchNormalization()(dense_1)
 dropout_1 = keras.layers.Dropout(0.4 )(norm_2)
 dense_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_1)
 norm_3 = keras.layers.BatchNormalization()(dense_2)
 dropout_2 = keras.layers.Dropout(0.4 )(norm_3)
 dropout_3 = keras.layers.Dropout(0.5 )(norm_2)
 dense_3 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='relu',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_3)
 norm_4 = keras.layers.BatchNormalization()(dense_3)
 dropout_4 = keras.layers.Dropout(0.4 )(norm_4)
 output_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
), name='main' )(dropout_4)
 output_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 Y_train_nonscored.shape[1],
activation='sigmoid',
bias_initializer=aux_output_bias 
), name='aux' )(dropout_2)
 model_nn = keras.models.Model(inputs=inp, outputs=[output_1, output_2])
 model_nn.compile(
 loss=[
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn is None else loss_fn,
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn_aux is None else loss_fn_aux
],
optimizer=keras.optimizers.Nadam(learning_rate=0.01),
metrics=['binary_crossentropy'],
loss_weights=[0.9, 0.1] 
)
 return model_nn
def fit(self, X, Y, X_valid, Y_valid, Y_aux, Y_aux_valid, max_epochs=100, save_weights=True):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 Y_aux = tf.cast(Y_aux, tf.float32)
 Y_aux_valid = tf.cast(Y_aux_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.7, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 class_weights_aux = balance_class_weights(Y_aux.numpy() , class_weights=(0.7, 1.4))
loss_fn_aux = weighted_binary_crossentropy(class_weights_aux, label_smoothing=1e-3)
 self._model_nn = self._create_nn_model(loss_fn=loss_fn, loss_fn_aux=loss_fn_aux)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=10,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_nn/weights_{repeat}_{fold}.h5',
monitor='val_main_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self._model_nn.fit(
 X, [Y, Y_aux],
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, [Y_valid, Y_aux_valid]),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self._model_nn.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)[0]
return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self._model_nn.save_weights('model_nn/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self._model_nn is None:
self._model_nn = self._create_nn_model()
if pretrained:
base_path = '/kaggle/input/model-nn-drug-cv/'
else:
base_path = '/kaggle/working/'
self._model_nn.load_weights(base_path + 'model_nn/weights_{}_{}.h5'.format(repeat, fold))
model.fit(x_train, y_train, batch_size = 512, epochs = 3, validation_data =(x_val, y_val), callbacks = [EarlyStopping(monitor='val_loss', min_delta = 0.0001)] )<predict_on_test>
n_repeats = 5
n_folds = 7
Y_test_preds_nn = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_nn = MultiLabelNN(n_input_features=X_test_nn.shape[1])
 model_nn.load_weights(repeat, fold)
 test_preds = model_nn.predict(X_test_nn)[0]
add_outputs(Y_test_preds_nn, test_preds, X=X_test_full)
 del model_nn
Y_test_preds_nn.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )
y_valpred = model.predict(x_val )<compute_test_metric>
def preprocess_tabnet(X, X_qtrans):
X_pca = pca_qtrans_2.transform(X_qtrans)
 g_std = X[gene_cols].std(axis=1 ).values.reshape(-1, 1)
 g_kurt = X[gene_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 g_skew = X[gene_cols].skew(axis=1 ).values.reshape(-1, 1)
 c_std = X[cell_cols].std(axis=1 ).values.reshape(-1, 1)
 c_kurt = X[cell_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 c_skew = X[cell_cols].skew(axis=1 ).values.reshape(-1, 1)
 X_std = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 return np.concatenate([
X_qtrans, X_pca,
g_std, g_kurt, g_skew,
c_std, c_kurt, c_skew,
], axis=1 )
f1 = 0
threshold = 0
for thresh in np.arange(0.1, 0.501,0.01):
f1score = np.round(metrics.f1_score(y_val,(y_valpred>thresh ).astype(int)) , 4)
 thresh = np.round(thresh,2)
 print('F1 score for threshold {} : {}'.format(thresh, f1score))
if f1score > f1:
f1 = f1score
threshold = thresh 
<predict_on_test>
X_test_tabnet = preprocess_tabnet(X_test, X_test_qtrans )
y_test = model.predict(x_test)
 y_test =(y_test[:,0] > threshold ).astype(np.int )<load_from_csv>
class MultiLabelTabNet(BaseEstimator):
def __init__(self, n_input_features, n_output_labels):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, loss_fn=None, output_bias=None):
if output_bias is not None:
output_bias = keras.initializers.Constant(output_bias)
 clf = StackedTabNetClassifier(
 feature_columns=None,
num_classes=1024,
num_layers=1,
feature_dim=2048,
output_dim=1024,
num_features=self.n_input_features,
num_decision_steps=1,
relaxation_factor=1.5,
sparsity_coefficient=1e-5,
norm_type='batch',
num_groups=-1,
output_activation='relu' 
)
 model = keras.models.Sequential([
keras.layers.InputLayer(self.n_input_features),
keras.layers.Dropout(0.5),
keras.layers.GaussianNoise(0.001),
clf,
keras.layers.Dropout(0.7),
tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
))
])
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(weight_decay=2.6e-5),
metrics=['binary_crossentropy'] 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=200):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 15.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 self.estimator_ = self._create_model(loss_fn=loss_fn, output_bias=initial_bias)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss', min_delta=1e-5,
patience=3, verbose=1, factor=0.1, min_lr=1e-7 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5, patience=10, restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_tabnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=[early_stop_cb, learning_rate_cb, checkpoint_cb],
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_tabnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model()
if pretrained:
base_path = '/kaggle/input/model-tabnet-drug-cv/'
else:
base_path = './'
self.estimator_.load_weights(base_path + f'model_tabnet/weights_{repeat}_{fold}.h5' )
test = pd.read_csv(".. /input/test.csv")['qid']<save_to_csv>
n_repeats = 5
n_folds = 7
Y_test_preds_tabnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_tabnet = MultiLabelTabNet(
 n_input_features=X_test_tabnet.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_tabnet.load_weights(repeat, fold)
 test_preds = model_tabnet.predict(X_test_tabnet)
 add_outputs(Y_test_preds_tabnet, test_preds, X=X_test_full)
 del model_tabnet
Y_test_preds_tabnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )
submit_df = pd.DataFrame({"qid": test, "prediction": y_test})
 submit_df.to_csv("submission.csv", index=False )<import_modules>
def preprocess_resnet(X):
X_scaled = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 X_genes = gene_pca_2.transform(X_scaled[gene_cols])
 X_cells = cell_pca_2.transform(X_scaled[cell_cols])
 return select_features(X), np.concatenate([X_genes, X_cells], axis=1 )
tqdm.pandas()<define_variables>
X_test_resnet_1, X_test_resnet_2 = preprocess_resnet(X_test )
NROWS = None
N_EMBED = 300
MAX_FEATURES = 95000
MAX_LEN = 72
N_SPLITS = 5
N_BATCH = 1021
N_EPOCH = 8
SEED = 1031<set_options>
class MultiLabelResNet(BaseEstimator):
def __init__(self, n_input_1, n_input_2, n_output_labels):
self.n_input_1 = n_input_1
self.n_input_2 = n_input_2
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, initial_bias, loss_fn=None):
output_bias = keras.initializers.Constant(initial_bias)
 inp_1 = keras.layers.Input(self.n_input_1, name='raw_inp')
 inp_2 = keras.layers.Input(self.n_input_2, name='pca_inp')
 gauss_2 = keras.layers.GaussianNoise(1e-4 )(inp_2)
 head_1 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(keras.layers.Dense(512, activation='elu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='elu'))
], name='head1')
 seq_1 = head_1(inp_1)
 seq_1_inp_concat = keras.layers.Concatenate()([gauss_2, seq_1])
 head_2 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "elu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "elu"))
], name='head2')
 seq_2 = head_2(seq_1_inp_concat)
 seq_1_seq_2_avg = keras.layers.Average()([seq_1, seq_2])
 head_3 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(206, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(self.n_output_labels, bias_initializer=output_bias, activation='sigmoid'))
], name='head3')
 output = head_3(seq_1_seq_2_avg)
 model = keras.models.Model(inputs=[inp_1, inp_2], outputs=output)
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(learning_rate=0.02, weight_decay=1e-5),
metrics=['binary_crossentropy'], 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=100, save_weights=True, initial_bias=initial_bias, transfer_model=None):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(initial_bias, loss_fn=loss_fn)
 if transfer_model is not None:
self.copy_weights(transfer_model)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=15,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_resnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model(initial_bias)
 if pretrained:
base_path = '/kaggle/input/model-resnet-drug-cv/'
else:
base_path = './'
self.estimator_.load_weights(base_path + 'model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def copy_weights(self, from_model):
for to_layer, from_layer in zip(self.estimator_.layers[:-1], from_model.estimator_.layers[:-1]):
to_layer.set_weights(from_layer.get_weights())
 for to_layer, from_layer in zip(self.estimator_.layers[-1].layers[:-1], from_model.estimator_.layers[-1].layers[:-1]):
to_layer.set_weights(from_layer.get_weights() )
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
n_repeats = 7
n_folds = 7
Y_test_preds_resnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 K.clear_session()
for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
model_resnet = MultiLabelResNet(
 n_input_1=X_test_resnet_1.shape[1],
n_input_2=X_test_resnet_2.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_resnet.load_weights(repeat, fold)
 test_preds = model_resnet.predict([X_test_resnet_1, X_test_resnet_2])
 add_outputs(Y_test_preds_resnet, test_preds, X=X_test_full)
 del model_resnet
Y_test_preds_resnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<define_variables>
weights = [0.35, 0.40, 0.25]
print(sum(weights))
assert(abs(sum(weights)- 1)< 1e-6)
 final_preds_drug_cv =(
 Y_test_preds_nn * weights[0] +
Y_test_preds_tabnet * weights[1] +
Y_test_preds_resnet * weights[2]
)
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<load_from_csv>
def preprocess_nn(X):
X_pca = pca_qtrans_2.transform(X)
 return np.concatenate([X, X_pca], axis=1 )
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv", nrows=NROWS)
 test_df = pd.read_csv(".. /input/test.csv", nrows=NROWS)
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 for df in [train_df, test_df]:
df["question_text"] = df["question_text"].apply(lambda x: x.lower())
 df["question_text"] = df["question_text"].apply(lambda x: clean_text(x))
df["question_text"] = df["question_text"].apply(lambda x: clean_numbers(x))
df["question_text"] = df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=MAX_LEN)
 test_X = pad_sequences(test_X, maxlen=MAX_LEN)
 train_y = train_df['target'].values
test_id = test_df["qid"].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index, test_id<statistical_test>
X_test_nn = preprocess_nn(X_test_qtrans )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
N_EMBED = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index)+ 1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, N_EMBED))
for word, i in word_index.items() :
if i >= MAX_FEATURES: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index)+ 1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, N_EMBED))
for word, i in word_index.items() :
if i >= MAX_FEATURES: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
N_EMBED = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index)+ 1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, N_EMBED))
for word, i in word_index.items() :
if i >= MAX_FEATURES: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<choose_model_class>
class MultiLabelNN(BaseEstimator):
def __init__(self,
n_input_features=num_features,
n_output_labels=num_labels,
n_hidden_units=2048):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.n_hidden_units = n_hidden_units
self._model_nn = None
def _create_nn_model(self, loss_fn=None, loss_fn_aux=None):
output_bias = keras.initializers.Constant(initial_bias)
 aux_output_bias = keras.initializers.Constant(nonscored_initial_bias)
 inp = keras.layers.Input(self.n_input_features)
 gauss_noise = keras.layers.GaussianNoise(0.001 )(inp)
 norm_1 = keras.layers.BatchNormalization()(gauss_noise)
 dense_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(norm_1)
 norm_2 = keras.layers.BatchNormalization()(dense_1)
 dropout_1 = keras.layers.Dropout(0.4 )(norm_2)
 dense_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_1)
 norm_3 = keras.layers.BatchNormalization()(dense_2)
 dropout_2 = keras.layers.Dropout(0.4 )(norm_3)
 dropout_3 = keras.layers.Dropout(0.5 )(norm_2)
 dense_3 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='relu',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_3)
 norm_4 = keras.layers.BatchNormalization()(dense_3)
 dropout_4 = keras.layers.Dropout(0.4 )(norm_4)
 output_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
), name='main' )(dropout_4)
 output_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 Y_train_nonscored.shape[1],
activation='sigmoid',
bias_initializer=aux_output_bias 
), name='aux' )(dropout_2)
 model_nn = keras.models.Model(inputs=inp, outputs=[output_1, output_2])
 model_nn.compile(
 loss=[
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn is None else loss_fn,
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn_aux is None else loss_fn_aux
],
optimizer=keras.optimizers.Nadam(learning_rate=0.01),
metrics=['binary_crossentropy'],
loss_weights=[0.9, 0.1] 
)
 return model_nn
def fit(self, X, Y, X_valid, Y_valid, Y_aux, Y_aux_valid, max_epochs=100, save_weights=True):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 Y_aux = tf.cast(Y_aux, tf.float32)
 Y_aux_valid = tf.cast(Y_aux_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.7, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 class_weights_aux = balance_class_weights(Y_aux.numpy() , class_weights=(0.7, 1.4))
loss_fn_aux = weighted_binary_crossentropy(class_weights_aux, label_smoothing=1e-3)
 self._model_nn = self._create_nn_model(loss_fn=loss_fn, loss_fn_aux=loss_fn_aux)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=10,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_nn/weights_{repeat}_{fold}.h5',
monitor='val_main_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self._model_nn.fit(
 X, [Y, Y_aux],
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, [Y_valid, Y_aux_valid]),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self._model_nn.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)[0]
return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self._model_nn.save_weights('model_nn/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self._model_nn is None:
self._model_nn = self._create_nn_model()
if pretrained:
base_path = '/kaggle/input/model-nn/'
else:
base_path = './working/'
self._model_nn.load_weights(base_path + 'model_nn/weights_{}_{}.h5'.format(repeat, fold))
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<normalization>
n_repeats = 5
n_folds = 7
Y_test_preds_nn = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_nn = MultiLabelNN(n_input_features=X_test_nn.shape[1])
 model_nn.load_weights(repeat, fold)
 test_preds = model_nn.predict(X_test_nn)[0]
add_outputs(Y_test_preds_nn, test_preds, X=X_test_full)
 del model_nn
Y_test_preds_nn.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
def preprocess_tabnet(X, X_qtrans):
X_pca = pca_qtrans_2.transform(X_qtrans)
 g_std = X[gene_cols].std(axis=1 ).values.reshape(-1, 1)
 g_kurt = X[gene_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 g_skew = X[gene_cols].skew(axis=1 ).values.reshape(-1, 1)
 c_std = X[cell_cols].std(axis=1 ).values.reshape(-1, 1)
 c_kurt = X[cell_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 c_skew = X[cell_cols].skew(axis=1 ).values.reshape(-1, 1)
 X_std = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 g_cluster = kmeans_g.predict(X_std[gene_cols] ).reshape(-1, 1)
 c_cluster = kmeans_c.predict(X_std[cell_cols] ).reshape(-1, 1)
 return np.concatenate([
X_qtrans, X_pca,
g_std, g_kurt, g_skew,
c_std, c_kurt, c_skew,
g_cluster, c_cluster
], axis=1 )
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 61
self.embedding = nn.Embedding(MAX_FEATURES, N_EMBED)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.11)
 self.lstm = nn.LSTM(N_EMBED, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, MAX_LEN)
 self.gru_attention = Attention(hidden_size * 2, MAX_LEN)
 self.linear = nn.Linear(hidden_size * 8, 17)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.11)
 self.out = nn.Linear(17, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<compute_test_metric>
X_test_tabnet = preprocess_tabnet(X_test, X_test_qtrans )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_test_metric>
class MultiLabelTabNet(BaseEstimator):
def __init__(self, n_input_features, n_output_labels):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, loss_fn=None, output_bias=None):
if output_bias is not None:
output_bias = keras.initializers.Constant(output_bias)
 clf = StackedTabNetClassifier(
 feature_columns=None,
num_classes=1024,
num_layers=1,
feature_dim=2048,
output_dim=1024,
num_features=self.n_input_features,
num_decision_steps=1,
relaxation_factor=1.5,
sparsity_coefficient=1e-5,
num_groups=-1,
output_activation='relu' 
)
 model = keras.models.Sequential([
keras.layers.InputLayer(self.n_input_features),
keras.layers.Dropout(0.5),
keras.layers.GaussianNoise(0.001),
clf,
keras.layers.Dropout(0.7),
tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
))
])
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(weight_decay=2.6e-5),
metrics=['binary_crossentropy'] 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=200):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 15.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(loss_fn=loss_fn, output_bias=initial_bias)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss', min_delta=1e-5,
patience=3, verbose=1, factor=0.1, min_lr=1e-7 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5, patience=10, restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_tabnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=[early_stop_cb, learning_rate_cb, checkpoint_cb],
shuffle=True,
verbose=2 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_tabnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model()
if pretrained:
base_path = '/kaggle/input/model-tabnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + f'model_tabnet/weights_{repeat}_{fold}.h5' )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<init_hyperparams>
n_repeats = 5
n_folds = 7
Y_test_preds_tabnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_tabnet = MultiLabelTabNet(
 n_input_features=X_test_tabnet.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_tabnet.load_weights(repeat, fold)
 test_preds = model_tabnet.predict(X_test_tabnet)
 add_outputs(Y_test_preds_tabnet, test_preds, X=X_test_full)
 del model_tabnet
Y_test_preds_tabnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )
start_time = time.time()
train_X, test_X, train_y, word_index, test_id = load_and_prec()
glove_embeddings = load_glove(word_index)
 fasttext_embeddings = load_fasttext(word_index)
 paragram_embeddings = load_para(word_index)
 total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
embedding_matrix = glove_embeddings * 0.33 + fasttext_embeddings * 0.34 + paragram_embeddings * 0.33
print(np.shape(embedding_matrix))
del glove_embeddings, fasttext_embeddings, paragram_embeddings
gc.collect()<split>
def preprocess_resnet(X):
X_scaled = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 X_genes = gene_pca_2.transform(X_scaled[gene_cols])
 X_cells = cell_pca_2.transform(X_scaled[cell_cols])
 return select_features(X), np.concatenate([X_genes, X_cells], axis=1 )
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED ).split(train_X, train_y))<categorify>
X_test_resnet_1, X_test_resnet_2 = preprocess_resnet(X_test )
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=N_BATCH, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr, step_size=step_size, mode='exp_range', gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=N_BATCH, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=N_BATCH, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(N_EPOCH):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * N_BATCH:(i+1)* N_BATCH] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, N_EPOCH, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * N_BATCH:(i+1)* N_BATCH] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<save_to_csv>
class MultiLabelResNet(BaseEstimator):
def __init__(self, n_input_1, n_input_2, n_output_labels):
self.n_input_1 = n_input_1
self.n_input_2 = n_input_2
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, initial_bias, loss_fn=None):
output_bias = keras.initializers.Constant(initial_bias)
 inp_1 = keras.layers.Input(self.n_input_1, name='raw_inp')
 inp_2 = keras.layers.Input(self.n_input_2, name='pca_inp')
 gauss_2 = keras.layers.GaussianNoise(1e-4 )(inp_2)
 head_1 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(keras.layers.Dense(512, activation='elu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='elu'))
], name='head1')
 seq_1 = head_1(inp_1)
 seq_1_inp_concat = keras.layers.Concatenate()([gauss_2, seq_1])
 head_2 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "elu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "elu"))
], name='head2')
 seq_2 = head_2(seq_1_inp_concat)
 seq_1_seq_2_avg = keras.layers.Average()([seq_1, seq_2])
 head_3 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(206, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(self.n_output_labels, bias_initializer=output_bias, activation='sigmoid'))
], name='head3')
 output = head_3(seq_1_seq_2_avg)
 model = keras.models.Model(inputs=[inp_1, inp_2], outputs=output)
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(learning_rate=0.02, weight_decay=1e-5),
metrics=['binary_crossentropy'], 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=100, save_weights=True, initial_bias=initial_bias, transfer_model=None):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(initial_bias, loss_fn=loss_fn)
 if transfer_model is not None:
self.copy_weights(transfer_model)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=15,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_resnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model(initial_bias)
 if pretrained:
base_path = '/kaggle/input/model-resnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + 'model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def copy_weights(self, from_model):
for to_layer, from_layer in zip(self.estimator_.layers[:-1], from_model.estimator_.layers[:-1]):
to_layer.set_weights(from_layer.get_weights())
 for to_layer, from_layer in zip(self.estimator_.layers[-1].layers[:-1], from_model.estimator_.layers[-1].layers[:-1]):
to_layer.set_weights(from_layer.get_weights() )
output = pd.DataFrame(data={"qid": test_id, "prediction": test_preds > 0.33})
 output.to_csv("submission.csv", index=False )<load_from_csv>
n_repeats = 7
n_folds = 7
Y_test_preds_resnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
model_resnet = MultiLabelResNet(
 n_input_1=X_test_resnet_1.shape[1],
n_input_2=X_test_resnet_2.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_resnet.load_weights(repeat, fold)
 test_preds = model_resnet.predict([X_test_resnet_1, X_test_resnet_2])
 add_outputs(Y_test_preds_resnet, test_preds, X=X_test_full)
 del model_resnet
Y_test_preds_resnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )
train = pd.read_csv(path + "/train.csv")
 test = pd.read_csv(path + "/test.csv")
 sample = pd.read_csv(path + "/sample_submission.csv")
 print("training data: " + str(train.shape))
print("test data: " + str(test.shape))
print("sample data: " + str(sample.shape)) 
<define_variables>
weights = [0.22, 0.43, 0.35]
print(sum(weights))
assert(abs(sum(weights)- 1)< 1e-6)
 final_preds_raw_cv =(
 Y_test_preds_nn * weights[0] +
Y_test_preds_tabnet * weights[1] +
Y_test_preds_resnet * weights[2]
)
<categorify><EOS> 
final_preds = 0.4 * final_preds_drug_cv + 0.6 * final_preds_raw_cv
final_preds.loc[X_test_full['cp_type'] != 'trt_cp', :] = 0
final_preds = np.concatenate([sig_id, final_preds], axis=1)
 final_preds = pd.DataFrame(final_preds, columns=Y_scored.columns)
 final_preds.to_csv('submission.csv', index=False )