Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<define_variables>
DO_SUBMISSION = True
DO_TRAIN_FOR_ENSEMBLE = False
DO_VIRTUAL_SUBMISSION = False
assert(sum([DO_SUBMISSION, DO_TRAIN_FOR_ENSEMBLE, DO_VIRTUAL_SUBMISSION])== 1), "select `ONE` mode"
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 preprocessed_dir = Path('.. /input/fat2019_prep_mels1' )<define_variables>
import os
import gc
import sys
import time
import shutil
from pathlib import Path
from copy import deepcopy
from contextlib import contextmanager
import random
import typing as tp
from collections import OrderedDict
import yaml
import numpy as np
import pandas as pd
from sklearn.metrics import log_loss
from sklearn.model_selection import train_test_split
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}<load_from_csv>
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 if CURATED_ONLY:
train_df = train_curated
else:
train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)
 train_df.head()<load_from_csv>
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()<define_variables>
%%time
!pip install --no-index --find-links.. /input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
num_classes = len(labels)
 num_classes<prepare_x_and_y>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape<load_pretrained>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 if CURATED_ONLY == False:
x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )<choose_model_class>
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
def create_model_inception(n_out=NUM_CLASSES):
base_model =InceptionV3(weights=None, include_top=False)
 x0 = base_model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 return model<define_search_model>
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
def create_model_xception(n_out=NUM_CLASSES):
base_model = Xception(weights=None, include_top=False)
 x0 = base_model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 return model<define_search_model>
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
def create_model_mobile(n_out=NUM_CLASSES):
base_model =MobileNetV2(weights=None, include_top=False)
 x0 = base_model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 return model<define_search_model>
def del_control(df: pd.DataFrame())-> pd.DataFrame() :
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 df['cp_type'] = train['cp_type']
df = df.loc[df['cp_type']=='trt_cp'].reset_index(drop=True)
 df = df.drop('cp_type', axis=1)
 return df
def drop_sig_id(df: pd.DataFrame())-> pd.DataFrame() :
return df.drop('sig_id', axis=1)
 def fix_oof(df: pd.DataFrame())-> pd.DataFrame() :
if df.shape[0] != 21948:
df = del_control(df)
 if 'sig_id' in df.columns.tolist() :
df = drop_sig_id(df)
 return df
def conv_simple_block(x, n_filters):
x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = AveragePooling2D()(x)
 return x
def create_model_simplecnn(n_out=NUM_CLASSES):
inp = Input(shape=(128,128,3))
x = conv_simple_block(inp,64)
 x = conv_simple_block(x,128)
 x = conv_simple_block(x,256)
 x = conv_simple_block(x,128)
 x = Flatten()(x)
 x = Dropout(0.2 )(x)
 x = Dense(128, activation='linear' )(x)
 x = PReLU()(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=inp, outputs=predictions)
 return model<define_search_model>
oof_list = []
def output_of_lambda(input_shape):
return(input_shape[0], input_shape[2], input_shape[3])
 def my_max(x):
return K.max(x, axis=1, keepdims=False)
 def crnn_simple_block(x, n_filters):
x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = Activation("relu" )(x)
 x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = Activation("relu" )(x)
 x = MaxPooling2D()(x)
 x = Dropout(0.2 )(x)
 return x
def create_model_crnn(n_out=NUM_CLASSES):
inp = Input(shape=(128,None,3))
x = crnn_simple_block(inp,64)
 x = crnn_simple_block(x,128)
 x = crnn_simple_block(x,256)
 x = Lambda(my_max, output_shape=output_of_lambda )(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = GlobalMaxPooling1D()(x)
 x = Dense(128, activation='linear' )(x)
 x = PReLU()(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=inp, outputs=predictions)
 return model<choose_model_class>
oof_list.append(fix_oof(
 pd.read_pickle('.. /input/nn-use-train-public/oof.pkl')))
 oof_list.append(fix_oof(
 pd.DataFrame(
 np.load('.. /input/tabnet-train-public-add-n-shared-1/oof_tabnet.npy'),
columns=TARGET_COL)) 
)
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/oof_prediction_takapy_resnet.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-seed-cv/oof_prediction.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-drug-seed-cv/oof_prediction.csv")))
def create_model_cnn8th(n_out=NUM_CLASSES):
regu=0
inp = Input(shape=(128,128,3))
x = Conv2D(48, 11, strides=(1,1),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(inp)
 x = BatchNormalization()(x)
 x = Conv2D(48, 11, strides=(2,3),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = MaxPooling2D(3, strides=(1,2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128, 5, strides=(1,1),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128, 5, strides=(2,3),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = MaxPooling2D(3, strides=2 )(x)
 x = BatchNormalization()(x)
 x = Conv2D(192, 3, strides=1,kernel_initializer='he_uniform', activation='relu', padding='same' )(x)
 x = BatchNormalization()(x)
 x = Conv2D(192, 3, strides=1,kernel_initializer='he_uniform', activation='relu', padding='same' )(x)
 x = BatchNormalization()(x)
 x = Conv2D(128, 3, strides=1,kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = MaxPooling2D(3, strides=(1,2))(x)
 x = BatchNormalization()(x)
 x = Flatten()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=inp, outputs=predictions)
 return model<choose_model_class>
y_true = fix_oof(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
K.clear_session()

 if MODEL == 'xception':
preprocess_input = preprocess_xception
model = create_model_xception(n_out=NUM_CLASSES)
 elif MODEL == 'inception':
preprocess_input = preprocess_inception
model = create_model_inception(n_out=NUM_CLASSES)
 elif MODEL == 'mobile':
preprocess_input = preprocess_mobile
model = create_model_mobile(n_out=NUM_CLASSES)
 elif MODEL == 'crnn':
preprocess_input = preprocess_mobile
model = create_model_crnn(n_out=NUM_CLASSES)
 elif MODEL == 'cnn8th':
preprocess_input = preprocess_mobile
model = create_model_cnn8th(n_out=NUM_CLASSES)
 else:
preprocess_input = preprocess_mobile
model = create_model_simplecnn(n_out=NUM_CLASSES)
 print(MODEL)
 model.summary()<load_from_csv>
print("sinchir0 - NN:", oof_list[0].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[0].values.ravel()))
display(oof_list[0].head())
 print("sinchir0 - TabNet:", oof_list[1].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[1].values.ravel()))
display(oof_list[1].head())
 print("takapy - ResNet:", oof_list[2].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[2].values.ravel()))
display(oof_list[2].head())
 print("tawara - ThrNN seed-cv:", oof_list[3].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[3].values.ravel()))
display(oof_list[3].head())
 print("tawara - ThrNN drug-seed-cv:", oof_list[4].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[4].values.ravel()))
display(oof_list[4].head() )
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_tf_lwlrap', factor=LR_FACTOR, patience=PATIENCE,
verbose=1, mode='max', min_delta=0.0001, cooldown=2, min_lr=1e-5)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 checkpoint = ModelCheckpoint(checkpoint_file[0], monitor='val_tf_lwlrap', verbose=1,
save_best_only=True, mode='max', save_weights_only = False)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]<split>
Blend_oof = oof_list[0].copy()
Blend_oof[:] = 0.
 for oof_df in oof_list:
Blend_oof += oof_df
Blend_oof = Blend_oof / len(oof_list )
x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)
 train_generator = FATTrainDataset.create_generator(
 x_trn, y_trn, BATCH_SIZE,(SIZE,SIZE,3), augument=TRAIN_AUGMENT, shuffling=True, mixup = USE_MIXUP, mixup_prob = MIXUP_PROB)
 validation_generator = FATTrainDataset.create_generator(
 x_val, y_val, BATCH_SIZE,(SIZE,SIZE,3), augument=VALID_AUGMENT, shuffling=False)
<data_type_conversions>
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 y_true = fix_oof(y_true)
 y_pred = Blend_oof.copy()
print("CV log_loss: ", log_loss(y_true.values.ravel() , y_pred.values.ravel()))
train_steps = np.ceil(float(len(x_trn)) / float(BATCH_SIZE))
val_steps = np.ceil(float(len(x_val)) / float(BATCH_SIZE))
train_steps = train_steps.astype(int)
 val_steps = val_steps.astype(int)
 print(train_steps, val_steps)
 print(len(x_trn)) 
<choose_model_class>
if DO_SUBMISSION or DO_VIRTUAL_SUBMISSION:
sub_list = [
pd.read_csv('./submission-sinchir0-nn.csv'),
pd.read_csv('./submission_sinchir0_tabnet.csv'),
pd.read_csv("./submission_takapy_tf-resnet.csv"),
pd.read_csv('./submission_tawara_thrnn_seed_cv.csv'),
pd.read_csv('./submission_tawara_thrnn_drug_seed_cv.csv'),
]
else:
sub_list = [
pd.read_csv('.. /input/nn-use-train-public/submission.csv'),
pd.read_csv('.. /input/tabnet-train-public-add-n-shared-1/submission.csv'),
pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/submission.csv"),
pd.read_csv('.. /input/moa-weight-thrnn-seed-cv/submission.csv'),
pd.read_csv('.. /input/moa-weight-thrnn-drug-seed-cv/submission.csv'),
]
print(LOSS)
 if LOSS=='BCEwithLogits':
model.compile(loss=BCEwithLogits,
optimizer=Adam(lr=LR),
metrics=[tf_lwlrap,'categorical_accuracy'])
 else:
model.compile(loss=LOSS,
optimizer=Adam(lr=LR),
metrics=[tf_lwlrap,'categorical_accuracy'])
<define_variables>
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )
print(LR, PATIENCE, LR_FACTOR,BATCH_SIZE, TRAIN_AUGMENT, USE_MIXUP, MIXUP_PROB )<train_model>
sub_list = [order_sub(sub_df)for sub_df in sub_list]
hist = model.fit_generator(
 train_generator,
steps_per_epoch=train_steps,
validation_data=validation_generator,
validation_steps=val_steps,
epochs=EPOCHS[0],
verbose=1,
callbacks=callbacks_list )<find_best_params>
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDataset2d(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][:, self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
print(K.eval(model.optimizer.lr))<train_model>
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )
if EPOCHS[1] > 0:
checkpoint = ModelCheckpoint(checkpoint_file[1], monitor='val_tf_lwlrap', verbose=1,
save_best_only=True, mode='max', save_weights_only = False)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
hist = model.fit_generator(
 train_generator,
steps_per_epoch=train_steps,
validation_data=validation_generator,
validation_steps=val_steps,
epochs=EPOCHS[1],
verbose=1,
callbacks=callbacks_list )<find_best_params>
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNForStacking(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int],
kwargs_head: tp.Dict, 
):

 super(CNNForStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(in_ch, out_ch, 3, 1, 0),
nn.BatchNorm1d(out_ch),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
print(K.eval(model.optimizer.lr))<train_model>
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True
if EPOCHS[2] > 0:
checkpoint = ModelCheckpoint(checkpoint_file[2], monitor='val_tf_lwlrap', verbose=1,
save_best_only=True, mode='max', save_weights_only = False)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
hist = model.fit_generator(
 train_generator,
steps_per_epoch=train_steps,
validation_data=validation_generator,
validation_steps=val_steps,
epochs=EPOCHS[2],
verbose=1,
callbacks=callbacks_list )<find_best_params>
def inference_function(settings, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in loader:
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr
print(K.eval(model.optimizer.lr))<predict_on_test>
X = np.concatenate([df.values for df in oof_list], axis=1)
 print(X.shape)
 y = y_true.values
print(y.shape)
 train_all_dataset = MoAStackingDataset(X, y)
 X_test = np.concatenate([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=1)
 print(X_test.shape)
 test_dataset = MoAStackingDataset(X_test, None)
 X_2d = np.stack([df.values for df in oof_list], axis=2)
 print(X_2d.shape)
 train_all_dataset_2d = MoAStackingDataset2d(X_2d, y)
 X_test_2d = np.stack(
 [sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 print(X_test_2d.shape)
 test_dataset_2d = MoAStackingDataset2d(X_test_2d, None )
model.load_weights(checkpoint_file[0])
 validation_generator = FATTrainDataset.create_generator(
 x_val, y_val, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False)
 pred_val_y = model.predict_generator(validation_generator,steps=val_steps,verbose=1)
 for kk in range(len(TTA)) :
for ii in range(TTA[kk]):
validation_generator = FATTrainDataset.create_generator(
 x_val, y_val, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False)
 pred_val_y += model.predict_generator(validation_generator,steps=val_steps,verbose=1)
 if kk+1 < len(TTA)and TTA[kk+1] > 0:
model.load_weights(checkpoint_file[kk+1])
 
<predict_on_test>
settings_str =
settings = yaml.safe_load(settings_str)
 settings["model"]["params"]["n_features_list"][0] = len(oof_list)* 206
train_generator = FATTrainDataset.create_generator(
 x_trn, y_trn, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False)
 pred_train_y = model.predict_generator(train_generator,steps=train_steps,verbose=1 )<compute_train_metric>
device = torch.device(settings["globals"]["device"])
 cv_score_list_mlp = []
stgs_list_mlp = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list_mlp.append(deepcopy(settings))
stgs_list_mlp[-1]["globals"]["seed"] = tmp_seed
def calculate_overall_lwlrap_sklearn(truth, scores):

 sample_weight = np.sum(truth > 0, axis=1)
 nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)
 overall_lwlrap = sklearn.metrics.label_ranking_average_precision_score(
 truth[nonzero_weight_sample_indices, :] > 0,
scores[nonzero_weight_sample_indices, :],
sample_weight=sample_weight[nonzero_weight_sample_indices])
 return overall_lwlrap<compute_test_metric>
oof_pred_arr_avg_mlp = np.zeros(( len(X), 206))
test_pred_arr_avg_mlp = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list_mlp):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
mlskf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=tmp_seed)
 train_val_indexs = list(mlskf.split(train_all_dataset.feat, train_all_dataset.label))
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = ".. /input/moa-weight-stacking-5model-by-mlp-seed-cv/best_model_fold{}_model{}.pth".format(fold_id, m_id)
 model = MLP(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg_mlp += oof_pred_arr
test_pred_arr_avg_mlp += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_list_mlp.append([m_id, "oof", oof_score])
 oof_pred_arr_avg_mlp /= len(stgs_list_mlp)
 test_pred_arr_avg_mlp /= len(stgs_list_mlp )
print("lwlrap from sklearn.metrics for training data =", calculate_overall_lwlrap_sklearn(y_trn, pred_train_y))
print("val lwlrap from sklearn.metrics =", calculate_overall_lwlrap_sklearn(y_val, pred_val_y/10))
score, weight = calculate_per_class_lwlrap(y_val, pred_val_y)
 lwlrap =(score * weight ).sum()
print('direct calculation of val lwlrap : %.4f' %(lwlrap))<normalization>
pd.DataFrame(cv_score_list_mlp, columns=["model", "fold", "val loss"] )
idx = np.sum(y_val,axis=1)> 1
print(y_val[idx, :].shape, y_val[idx==False, :].shape)
 print("val lwlrap for multi-labels =", calculate_overall_lwlrap_sklearn(y_val[idx], pred_val_y[idx]))
print("val lwlrap for single-label =", calculate_overall_lwlrap_sklearn(y_val[idx==False], pred_val_y[idx==False]))<data_type_conversions>
print(log_loss(y.ravel() , oof_pred_arr_avg_mlp.ravel()))
test_steps = np.ceil(float(len(x_test)) / float(BATCH_SIZE)).astype(int)
<predict_on_test>
settings_str =
settings = yaml.safe_load(settings_str)
 settings["model"]["params"]["n_models"] = len(oof_list )
model.load_weights(checkpoint_file[0])
 test_generator = FATTrainDataset.create_generator(
 x_test, x_test, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False, test_data=True)
 pred_test_y = model.predict_generator(test_generator,steps=test_steps,verbose=1)
 for kk in range(len(TTA)) :
for ii in range(TTA[kk]):
test_generator = FATTrainDataset.create_generator(
 x_test, x_test, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False, test_data=True)
 pred_test_y += model.predict_generator(test_generator,steps=test_steps,verbose=1)
 if kk+1 < len(TTA)and TTA[kk+1] > 0:
model.load_weights(checkpoint_file[kk+1] )<data_type_conversions>
device = torch.device(settings["globals"]["device"])
 cv_score_list_cnn = []
stgs_list_cnn = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list_cnn.append(deepcopy(settings))
stgs_list_cnn[-1]["globals"]["seed"] = tmp_seed
sort_idx = np.argsort(labels ).astype(int)
<sort_values>
oof_pred_arr_avg_cnn = np.zeros(( len(X), 206))
test_pred_arr_avg_cnn = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list_cnn):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
mlskf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=tmp_seed)
 train_val_indexs = list(mlskf.split(train_all_dataset.feat, train_all_dataset.label))
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
train_all_dataset_2d.reset_model_order()
train_all_dataset_2d.shuffle_model_order(tmp_seed)
 test_dataset_2d.model_order = train_all_dataset_2d.model_order
print("[model {}'s order]: {}".format(m_id, train_all_dataset_2d.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = ".. /input/moa-weight-stacking-5model-by-1dcnn-seed-cv/best_model_cnn_fold{}_model{}.pth".format(fold_id, m_id)
 model = CNNForStacking(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset_2d, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset_2d, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg_cnn += oof_pred_arr
test_pred_arr_avg_cnn += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_list_cnn.append([m_id, "oof", oof_score])
 oof_pred_arr_avg_cnn /= len(stgs_list_cnn)
 test_pred_arr_avg_cnn /= len(stgs_list_cnn )
print(sort_idx )<save_to_csv>
pd.DataFrame(cv_score_list_cnn, columns=["model", "fold", "val loss"] )
sample_sub = pd.read_csv('.. /input/freesound-audio-tagging-2019/sample_submission.csv')
 test_Y_sort = pred_test_y[:, sort_idx]
sample_sub.iloc[:, 1:] = test_Y_sort
sample_sub.to_csv('submission.csv', index=False)
 sample_sub.head()<set_options>
print(log_loss(y.ravel() , oof_pred_arr_avg_cnn.ravel()))
gc.enable() 
<load_from_csv>
opt_weights = [ 0.1709929 , 0.22160569, 0.07207321, 0.95431566, -0.42108426]
def feature_engineering(is_train=True):
if is_train:
print("processing train.csv")
 df = pd.read_csv(".. /input/train_V2.csv")
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv(".. /input/test_V2.csv")
 df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
print("remove some columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
print("get target")
 if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group sum feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('sum')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_sum", "_sum_rank"], how='left', on=['matchId', 'groupId'])
 df_out=reduce_mem_usage(df_out)
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 df_out=reduce_mem_usage(df_out)
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out=reduce_mem_usage(df_out)
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out)
 feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names 
<prepare_x_and_y>
oof_pred_arr_avg_wo = np.zeros(( len(X),206))
for w, oof_df in zip(opt_weights, oof_list):
oof_pred_arr_avg_wo += oof_df.values * w
print(log_loss(y.ravel() , oof_pred_arr_avg_wo.ravel()))
x_train, y, feature_names = feature_engineering(True )<split>
BLEND_wo = sub_list[0].copy()
BLEND_wo.iloc[:, 1:] = 0.
 for w, sub_df in zip(opt_weights, sub_list):
BLEND_wo.iloc[:,1:] += sub_df.iloc[:,1:] * w
X_train, X_val, y_train, y_val = train_test_split(x_train, y, test_size=0.33, random_state=42)
<set_options>
oof_pred_arr_avg =(oof_pred_arr_avg_mlp + oof_pred_arr_avg_cnn + oof_pred_arr_avg_wo)/ 3
print(log_loss(y.ravel() , oof_pred_arr_avg.ravel()))
warnings.filterwarnings("ignore")
 color = sns.color_palette() 
<predict_on_test>
BLEND = sub_list[0].copy()
BLEND.iloc[:, 1:] =(BLEND_mlp.iloc[:, 1:] + BLEND_cnn.iloc[:, 1:] + BLEND_wo.iloc[:,1:])/ 3
BLEND.head()
train_data=lgb.Dataset(X_train, label=y_train)
 val_data= lgb.Dataset(X_val, label=y_val)
 params = {
'num_leaves': 144,
'learning_rate': 0.1,
'n_estimators': 1500,
'max_depth':12,
'max_bin':55,
'bagging_fraction':0.8,
'bagging_freq':5,
'feature_fraction':0.9,
'verbose':50,
'early_stopping_rounds':100
}
params['metric'] = 'auc'
light_reg= lgb.train(params, train_data, valid_sets=val_data, num_boost_round=5000, early_stopping_rounds=100)
 y_pred=light_reg.predict(X_val)
 print(mean_absolute_error(y_val,y_pred)) 
<predict_on_test>
if DO_SUBMISSION or DO_TRAIN_FOR_ENSEMBLE:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 elif DO_VIRTUAL_SUBMISSION:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 print(df_test.shape, submission.shape)
 df_test, submission = generate_virtual_private(df_test, submission)
 print(df_test.shape, submission.shape)
 else:
raise ValueError
<prepare_x_and_y><EOS> 
df = submission.copy()
public_id = list(df['sig_id'].values)
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type =='ctl_vehicle'].sig_id] = 0
df_submit.to_csv('submission.csv',index=True )