Kaggle competition: Mechanisms of Action (MoA) Prediction
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand_name': np.array(dataset.brand_name)
 ,'category_name': np.array(dataset.category_name)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )<compute_train_metric>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.15
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 50 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category_name)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128 )(main_l))
main_l = Dropout(dr_r )(Dense(64 )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name
, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()
5<train_model>
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
BATCH_SIZE = 10000
epochs = 3
model = get_model()
model.fit(X_train, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid, dvalid.target)
 , verbose=1 )<compute_train_metric>
n_labels = train_targets_scored.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
val_preds = model.predict(X_valid)
 val_preds = target_scaler.inverse_transform(val_preds)
 val_preds = np.exp(val_preds)+1
y_true = np.array(dvalid.price.values)
 y_pred = val_preds[:,0]
v_rmsle = rmsle(y_true, y_pred)
 print(" RMSLE error on dev test: "+str(v_rmsle))<predict_on_test>
def build_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model
preds = model.predict(X_test, batch_size=BATCH_SIZE)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1
submission = test[["test_id"]]
submission["price"] = preds<save_to_csv>
n_seeds = 7
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 10
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
kf = KFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in kf.split(train_features):
X_train, X_test = preprocessor(train_features.iloc[train].values,
train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values)
 X_train_2 = train_features.iloc[train][predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = build_model(n_features, n_features_2, n_labels)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=2, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=10, mode='min',restore_best_weights=True)
 hist = model.fit([X_train,X_train_2],y_train, batch_size=128, epochs=192,verbose=0,validation_data =([X_test,X_test_2],y_test),
callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('TwoHeads_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict([X_test,X_test_2])
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1
submission.to_csv("./myNNsubmission.csv", index=False)
 submission.price.hist() 
<feature_engineering>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = y_pred
sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('sub2H.csv', index=False )
start_time = time.time()
os.environ['MKL_NUM_THREADS'] = '4'
os.environ['OMP_NUM_THREADS'] = '4'
start_time = time.time()<data_type_conversions>
import os
import gc
import pickle
import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from tqdm.notebook import tqdm
from time import time
def preprocess_RNN(dataset):
print("Filling Missing Values")
 dataset['category_name'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 print("Casting data types to type Category")
 dataset['category_name'] = dataset['category_name'].astype('category')
 dataset['brand_name'] = dataset['brand_name'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category')
 print("RNN PreProcessing completed")
 return dataset
def fit_RNN_text(dataset):
print("Fit Name and Item description fields for Tokenization")
 raw_text = np.hstack([dataset.item_description.str.lower() , dataset.name.str.lower() ])
 word_token = Tokenizer()
word_token.fit_on_texts(raw_text)
 print("RNN Data fit completed")
 return word_token
def fit_RNN_label(dataset):
print("Fit Categorical variables on full Merged Test and Train data")
 le_name = LabelEncoder()
le_name.fit(dataset.category_name)
 le_brand = LabelEncoder()
le_brand.fit(dataset.brand_name)
 print("Completed Label fitting")
 return le_name, le_brand
def transform_RNN(dataset, le_name, le_brand, word_token):
print("Use Defined Label encoders to Encode brand and category_name")
 dataset['category'] = le_name.transform(dataset.category_name)
 dataset['brand'] = le_brand.transform(dataset.brand_name)
 print("Convert Text to sequences")
 dataset["seq_item_description"] = word_token.texts_to_sequences(dataset.item_description.str.lower())
 dataset["seq_name"] = word_token.texts_to_sequences(dataset.name.str.lower())
 print("Sequence Conversion Completed")
 return dataset
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description
, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand': np.array(dataset.brand)
 ,'category': np.array(dataset.category)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
print("Data ready for Vectorization")
 return X<choose_model_class>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
scaler = StandardScaler()
X = scaler.fit_transform(train.values)
 x_tt = scaler.transform(test_features.values)
 N_STARTS = 3
N_SPLITS = 5
res_svc = train_targets.copy()
ss_svc.loc[:, train_targets.columns] = 0
res_svc.loc[:, train_targets.columns] = 0
for tar in tqdm(range(train_targets.shape[1])) :
start_time = time()
targets = train_targets.values[:, tar]
if targets.sum() >= N_SPLITS:
for seed in range(N_STARTS):
skf = StratifiedKFold(n_splits = N_SPLITS, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(targets, targets)) :
x_tr, x_val = X[tr], X[te]
y_tr, y_val = targets[tr], targets[te]
model = Ridge(alpha = 0.0001)
 model.fit(x_tr, y_tr)
 ss_svc.loc[:, train_targets.columns[tar]] += model.predict(x_tt)/(N_SPLITS * N_STARTS)
 res_svc.loc[te, train_targets.columns[tar]] += model.predict(x_val)/ N_STARTS
score = log_loss(train_targets.loc[:, train_targets.columns[tar]], res_svc.loc[:, train_targets.columns[tar]])
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] SVM Target {tar}:', score)
 print(f'SVM OOF Metric: {log_loss_metric(train_targets, res_svc)}')
 res_svc.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_svc.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'SVM OOF Metric with postprocessing: {log_loss_metric(train_targets, res_svc)}' )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs )<categorify>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
def RNN_model() :
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand = Input(shape=[1], name="brand")
 category = Input(shape=[1], name="category")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_size = 60
emb_name = Embedding(MAX_TEXT, emb_size//3 )(name)
 emb_item_desc = Embedding(MAX_TEXT, emb_size )(item_desc)
 emb_brand = Embedding(MAX_BRAND, 10 )(brand)
 emb_category = Embedding(MAX_CATEGORY, 10 )(category)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(25 )(emb_item_desc)
 rnn_layer2 = GRU(12 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand)
 , Flatten()(emb_category)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = BatchNormalization()(main_l)
 main_l = Dropout(0.1 )(Dense(512,activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(64,activation='relu' )(main_l))
output = Dense(1,activation="linear" )(main_l)
 model = Model([brand, category, item_condition, item_desc, name, num_vars], output)
 optimizer = optimizers.Adam()
model.compile(loss="mse",
optimizer=optimizer)
 return model
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 \
for i, pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5
def eval_model(model):
val_preds = model.predict(X_valid)
 val_preds = np.expm1(val_preds)
 y_pred = val_preds[:, 0]
y_true = np.array(valid_prices)
 yt = pd.DataFrame(y_true)
 yp = pd.DataFrame(y_pred)
 print(yt.isnull().any())
 print(yp.isnull().any())
 v_rmsle = rmsle(y_true, y_pred)
 print(" RMSLE error on dev test: "+str(v_rmsle))
return v_rmsle
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1<feature_engineering>
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 remove_vehicle = True
if remove_vehicle:
train_features = train.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 else:
train_features = train
t1 = time.time()
train = pd.read_table('.. /input/mercari-price-suggestion-challenge/train.tsv', sep='\t')
 train = train[train['price'] !=0 ]
ntrain = train.shape[0]
train['target'] = np.log1p(train['price'])
 test = pd.read_table('.. /input/mercari-price-suggestion-challenge/test_stg2.tsv', sep='\t')
 ntest = test.shape[0]
merge = pd.concat([train, test], 0, ignore_index = True)
 del test, train
gc.collect()<categorify>
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42)
 MAX_EPOCH=250
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux)
 data_path = ".. /input/lish-moa/"
t1 = time.time()
merge = preprocess_RNN(merge)
 t2 = time.time()
print("Time taken for RNN preprocess "+str(t2-t1))
print(merge.shape)
 t1 = time.time()
le_name, le_brand = fit_RNN_label(merge)
 t2 = time.time()
print("Time taken to Fit on RNN "+str(t2-t1))
t1 = time.time()
train = merge[:ntrain]
word_token = fit_RNN_text(train)
 train = transform_RNN(train, le_name, le_brand, word_token)
 t2 = time.time()
print("Time taken to Fit and Transform on RNN "+str(t2-t1))<define_variables>
train = res_svc[cols].values
X_test = ss_svc[cols].values
train_targets_scored = pd.read_csv(data_path+'train_targets_scored.csv')
 train_targets_scored.drop(columns=["sig_id"], inplace=True)
 submission = pd.read_csv(data_path+'sample_submission.csv')
 scores_auc_all= []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, random_state=0, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, train_targets_scored)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train[train_idx, :], train_targets_scored.values[train_idx, :]
X_val, y_val = train[val_idx, :], train_targets_scored.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds)
 aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )
MAX_NAME_SEQ = 20
MAX_ITEM_DESC_SEQ = 60
MAX_CATEGORY_NAME_SEQ = 20
MAX_CATEGORY = np.max(train.category.max())+1
MAX_BRAND = np.max(train.brand.max())+1
MAX_CONDITION = 6
MAX_TEXT = np.max([np.max(train.seq_name.max()), np.max(train.seq_item_description.max())])+2<split>
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
submission[all_feat] = test_preds_all.mean(axis=0)
 submission.loc[test['cp_type']==1, submission.columns[1:]] = 0
submission.to_csv('subSVM.csv', index=None)
 subSVM = submission.copy()
dtrain, dvalid = train_test_split(train, random_state=233, train_size=0.99)
 dtrain['target'] =np.log1p(dtrain['price'])
 target = np.array(dtrain.target)
 dvalid['target'] =np.log1p(dvalid['price'])
 valid_prices = np.array(dvalid.target)
 print(dvalid.shape, dtrain.shape )<prepare_x_and_y>
train = oof
X_test = y_pred
train_targets_scored = pd.read_csv(data_path+'train_targets_scored.csv')
 train_targets_scored.drop(columns=["sig_id"], inplace=True)
 submission = pd.read_csv(data_path+'sample_submission.csv')
 scores_auc_all= []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, random_state=0, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, train_targets_scored)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train[train_idx, :], train_targets_scored.values[train_idx, :]
X_val, y_val = train[val_idx, :], train_targets_scored.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds)
 aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )
del train
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 del dtrain, dvalid
gc.collect()<choose_model_class>
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
submission[all_feat] = test_preds_all.mean(axis=0)
 submission.loc[test['cp_type']==1, submission.columns[1:]] = 0
submission.to_csv('subTab2.csv', index=None)
 subTab = submission.copy()
<train_model><EOS> 
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 predictions =[]
predictions.append(subSVM)
 predictions.append(sub)
 predictions.append(subTab)
 columns = list(submission.columns)
 columns.remove('sig_id')
 y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 public_ids = list(submission['sig_id'].values)
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
columns = list(submission.columns)
 columns.remove('sig_id')
 submission = pd.DataFrame(index = public_ids + private_ids, columns=columns)
 submission.index.name = 'sig_id'
submission[:] = 0
p_min = 0.0005
p_max = 0.9995
submission.loc[y_pred.sig_id,:] = np.clip(y_pred[columns].values,p_min,p_max)
 submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
 submission.head().T