Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<define_variables>
DO_SUBMISSION = True
DO_TRAIN_FOR_ENSEMBLE = False
DO_VIRTUAL_SUBMISSION = False
assert(sum([DO_SUBMISSION, DO_TRAIN_FOR_ENSEMBLE, DO_VIRTUAL_SUBMISSION])== 1), "select `ONE` mode"
def get_feed_dict(batch):
if batch.shape[1] == 98:
feed_dict={ desc_indices: batch[:, 0:75],
info_indices: batch[:, 75:95],
desc_seq_length: batch[:, 95],
info_seq_length: batch[:, 96],
log_price: batch[:, 97:98]}
else:
feed_dict={ desc_indices: batch[:, 0:75],
info_indices: batch[:, 75:95],
desc_seq_length: batch[:, 95],
info_seq_length: batch[:, 96],
}
return feed_dict<predict_on_test>
%%time
!pip install --no-index --find-links.. /input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
def predict(session, data, batch_size):
start = 0
for step in range(len(data)//batch_size + 1):
batch, start = get_batch(data, start, batch_size)
 pred_ = sess.run(pred, feed_dict=get_feed_dict(batch))
if step == 0:
pred_all = np.array(pred_)
 else:
pred_all = np.vstack(( pred_all, pred_))
assert len(pred_all)== len(data)
 return pred_all<init_hyperparams>
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/
train_start, valid_start, time_initial = 0, 0, time()
batch_size, epochs = 500, 2
no_batches = len(train)//batch_size + 1
avg_loss, time_0 = 0, time()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True), graph=graph)as sess:
init.run()
for epoch in range(epochs):
for step in range(no_batches):
if step % 1000 == 0: print(step)
 batch, train_start = get_batch(train, train_start, batch_size)
 _, loss_step = sess.run([train_op, loss], feed_dict=get_feed_dict(batch))
avg_loss += loss_step
avg_loss /= no_batches
print("epoch: %2d, average loss: %.3f, time: %2.2f"
%(epoch, avg_loss, time() -time_0))
avg_loss, time_0 = 0, time()
valid_pred = predict(sess, valid, batch_size)
 valid_log_price = valid[:, 97:98]
valid_rsme = np.sqrt(np.mean(( valid_pred - valid_log_price)**2))
print("validation rsme: %.4f" %valid_rsme)
 save_path = saver.save(sess,"./params.ckpt")
 print("saved parameters to: ", save_path)
 print("done training")
 print("total training time is: %4d" %(time() - time_initial))<load_from_csv>
warnings.resetwarnings()
warnings.simplefilter('ignore', FutureWarning)
 warnings.simplefilter('ignore', DeprecationWarning)
 os.environ["CUDA_VISIBLE_DEVICES"]="0"
del train, valid
test = pd.read_csv(".. /input/test_stg2.tsv", sep='\t')
 test = prepare_data(test)
 test = to_array(test)
 print(test.shape )<prepare_output>
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True), graph=graph)as sess:
saver.restore(sess, "./params.ckpt")
 test_pred = predict(sess, test, batch_size)
 prices = np.exp(test_pred)- 1
assert len(test_pred)== len(test)
 out = pd.DataFrame(prices)
 out = out.reset_index().rename(columns={"index":"test_id", 0:"price"} )<save_to_csv>
MODEL_NAMES = ["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]
out.to_csv("./out.csv", index=False)
 print("output saved to csv" )<predict_on_test>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
SUBMISSION = True
print("SUBMISSION: {}".format(SUBMISSION))<feature_engineering>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
os.environ['MKL_NUM_THREADS'] = '4'
os.environ['OMP_NUM_THREADS'] = '4'
os.environ['JOBLIB_START_METHOD'] = 'forkserver'<import_modules>
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
import gc
import re
import math
from time import time
from collections import Counter
import tensorflow as tf
import pandas as pd
import numpy as np
import csv
from fastcache import clru_cache as lru_cache
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords<compute_test_metric>
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)- np.log1p(y0), 2)) )<load_from_csv>
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
print('Loading GloVE...')
 GLOVE_PATH = '.. /input/glove-global-vectors-for-word-representation/glove.6B.50d.txt'
embeddings_df = pd.read_table(GLOVE_PATH, sep=" ", index_col=0, header=None, quoting=csv.QUOTE_NONE)
 word_embeddings_matrix = embeddings_df.values.astype(np.float32)
 print('GloVE Word embeddings shape:', word_embeddings_matrix.shape)
 word_embedding_vocab = {t: i for(i, t)in enumerate(embeddings_df.index.tolist())}
del(embeddings_df)
 print_elapsed()<load_from_csv>
def del_control(df: pd.DataFrame())-> pd.DataFrame() :
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 df['cp_type'] = train['cp_type']
df = df.loc[df['cp_type']=='trt_cp'].reset_index(drop=True)
 df = df.drop('cp_type', axis=1)
 return df
def drop_sig_id(df: pd.DataFrame())-> pd.DataFrame() :
return df.drop('sig_id', axis=1)
 def fix_oof(df: pd.DataFrame())-> pd.DataFrame() :
if df.shape[0] != 21948:
df = del_control(df)
 if 'sig_id' in df.columns.tolist() :
df = drop_sig_id(df)
 return df
print('Reading train data...')
 df_train = pd.read_table('.. /input/mercari-price-suggestion-challenge/train.tsv', engine='c')
 print('Train set size: {}'.format(len(df_train)))
 print_elapsed()<define_variables>
oof_list = []
oof_list.append(fix_oof(
 pd.read_pickle('.. /input/nn-use-train-public/oof.pkl')))
 oof_list.append(fix_oof(
 pd.DataFrame(
 np.load('.. /input/tabnet-train-public-add-n-shared-1/oof_tabnet.npy'),
columns=TARGET_COL)) 
)
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/oof_prediction_takapy_resnet.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-seed-cv/oof_prediction.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-drug-seed-cv/oof_prediction.csv")) )
print('Generating features with statistics for item description textual content')
 acronyms_regex = re.compile('([A-Z\-0-9]{2,})')
 hashtag_regex = re.compile(r'(
 def extract_counts(text):
text_size_words_counts = len(text.split(' '))
text_size_words_log_counts = math.log1p(text_size_words_counts)
 full_uppercase_perc = len(acronyms_regex.findall(text)) / float(text_size_words_counts)
 exclamation_log_count = math.log1p(text.count('!'))
star_log_count = math.log1p(text.count('*'))
percentage_log_count = math.log1p(text.count('%'))
price_removed_marker_log_count = math.log1p(text.count('[rm]'))
hashtag_log_count = math.log1p(len(hashtag_regex.findall(text)))
 return [text_size_words_log_counts,
full_uppercase_perc,
exclamation_log_count,
star_log_count,
percentage_log_count,
price_removed_marker_log_count,
hashtag_log_count]
item_descr_counts = np.vstack(df_train['item_description'].astype(str ).apply(extract_counts ).values)
 item_descr_counts_scaler = StandardScaler(copy=True)
 X_item_descr_counts = item_descr_counts_scaler.fit_transform(item_descr_counts)
 del(item_descr_counts)
 print_elapsed()<drop_column>
y_true = fix_oof(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
for i, name in enumerate(MODEL_NAMES):
print(f"[{name}]")
 print("shape:", oof_list[i].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[i].values.ravel()))
price = df_train.pop('price' )<randomize_order>
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
class MoAStackingDatasetForCNN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][..., self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDatasetForGCN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
print('Spliting train/validation set')
 valid_rate = 0.00001 if SUBMISSION else 0.1
valid_size = int(len(df_train)*valid_rate)
 np.random.seed(100)
 rows_idxs = np.arange(0,len(df_train))
np.random.shuffle(rows_idxs)
 valid_idxs = rows_idxs[-valid_size:]
train_zeroed_prices_idxs = price.iloc[np.in1d(price.index.values, valid_idxs, invert=True)][price < 3.0].index.values
train_idxs = price.iloc[np.in1d(price.index.values, np.hstack([valid_idxs, train_zeroed_prices_idxs]),
invert=True)].index.values
assert len(df_train)== len(train_zeroed_prices_idxs)+ len(train_idxs)+ len(valid_idxs)
 assert len(set(train_idxs ).intersection(set(valid_idxs)))== 0
train_size = len(train_idxs)
 del(rows_idxs)
 print_elapsed()<prepare_x_and_y>
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )
print('Normalizing price')
 price_log = np.log1p(price)
 price_log_train = price_log.iloc[train_idxs].values
price_log_train_mean = price_log_train.mean()
price_log_train_std = price_log_train.std()
del(price_log_train)
 y =(price_log - price_log_train_mean)/ price_log_train_std
y = y.values.reshape(-1, 1)
 print_elapsed()<groupby>
def vector_wise_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 X = torch.transpose(X, 0, 1)
 W = torch.transpose(W, 1, 2)
 H = torch.matmul(X, W)
 H = torch.transpose(H, 0, 1)
 return H
def vector_wise_shared_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 H = nn.functional.linear(X, W)
 return H
print('Gessing null Brands from name and category...')
 def concat_categories(x):
return set(x.values)
 brand_names_categories = dict(df_train[df_train['brand_name'] != 'unk_brand'][['brand_name','category_name']].astype('str' ).groupby('brand_name' ).agg(concat_categories ).reset_index().values.tolist())
 brands_sorted_by_size = list(sorted(filter(lambda y: len(y)>= 3, list(brand_names_categories.keys())) , key = lambda x: -len(x)))
 brand_name_null_count = len(df_train.loc[df_train['brand_name'] == 'unk_brand'])
 def brandfinder(name, category):
for brand in brands_sorted_by_size:
if brand in name and category in brand_names_categories[brand]:
return brand
return 'unk_brand'
train_names_unknown_brands = df_train[df_train['brand_name'] == 'unk_brand'][['name','category_name']].astype('str' ).values
train_estimated_brands = Parallel(n_jobs=4 )(delayed(brandfinder )(name, category)for name, category in train_names_unknown_brands)
 df_train.ix[df_train['brand_name'] == 'unk_brand', 'brand_name'] = train_estimated_brands
found = brand_name_null_count-len(df_train.loc[df_train['brand_name'] == 'unk_brand'])
 print("Null brands found: %d from %d" %(found, brand_name_null_count))
print_elapsed()<groupby>
def _calculate_fan_in_and_fan_out_for_vwl(tensor)-> tp.Tuple[int]:

 dimensions = tensor.dim()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
 fan_in = tensor.size(-1)
 fan_out = tensor.size(-2)
 return fan_in, fan_out
def _calculate_correct_fan_for_vwl(tensor, mode)-> int:

 mode = mode.lower()
valid_modes = ['fan_in', 'fan_out']
if mode not in valid_modes:
raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
fan_in, fan_out = _calculate_fan_in_and_fan_out_for_vwl(tensor)
 return fan_in if mode == 'fan_in' else fan_out
def kaiming_uniform_for_vwl(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'):

 fan = _calculate_correct_fan_for_vwl(tensor, mode)
 gain = nn.init.calculate_gain(nonlinearity, a)
 std = gain / np.sqrt(fan)
 bound = np.sqrt(3.0)* std
with torch.no_grad() :
return tensor.uniform_(-bound, bound )
print('Generating features from category statistics for price...')
 CAT_STATS_MIN_COUNT = 5
STD_SIGMAS = 2
df_train['price_log'] = price_log
cats_stats_df = df_train.iloc[train_idxs].groupby(['category_name', 'brand_name', 'shipping'] ).agg({'category_name': len,
'price_log': [np.median, np.mean, np.std]})
 cats_stats_df.columns = ['price_log_median', 'price_log_mean', 'price_log_std','count']
cats_stats_df.drop(cats_stats_df[cats_stats_df['count'] < CAT_STATS_MIN_COUNT].index, inplace=True)
 cats_stats_df['price_log_std'] = cats_stats_df['price_log_std'].fillna(0)
 cats_stats_df['price_log_conf_variance'] = cats_stats_df['price_log_std'] / cats_stats_df['price_log_mean']
cats_stats_df['count_log'] = np.log1p(cats_stats_df['count'])
 cats_stats_df['min_expected_log_price'] =(cats_stats_df['price_log_mean'] - cats_stats_df['price_log_std']*STD_SIGMAS ).clip(lower=1.0)
 cats_stats_df['max_expected_log_price'] =(cats_stats_df['price_log_mean'] + cats_stats_df['price_log_std']*STD_SIGMAS)
 del(df_train['price_log'])
 len(cats_stats_df)
 def merge_with_cat_stats(df):
return df.merge(cats_stats_df.reset_index() , how='left',
on=['category_name', 'brand_name', 'shipping'])[['price_log_median', 'price_log_mean', 'price_log_std',
'price_log_conf_variance', 'count_log', 'min_expected_log_price', 'max_expected_log_price']].fillna(0 ).values
train_cats_stats_features = merge_with_cat_stats(df_train)
 cats_stats_features_scaler = StandardScaler(copy=True)
 X_cats_stats_features_scaled = cats_stats_features_scaler.fit_transform(train_cats_stats_features)
 del(train_cats_stats_features)
 print_elapsed()<set_options>
class VectorWiseLinear(nn.Module):

 def __init__(
 self,
in_features: int, out_features: int, n_vectors: int,
bias: bool=True, weight_shared: bool=True 
)-> None:

 super(VectorWiseLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.n_vectors = n_vectors
self.weight_shared = weight_shared
if self.weight_shared:
self.weight = nn.Parameter(
 torch.Tensor(self.out_features, self.in_features))
self.matmul_func = vector_wise_shared_matmul
else:
self.weight = nn.Parameter(
 torch.Tensor(self.n_vectors, self.out_features, self.in_features))
self.matmul_func = vector_wise_matmul
if bias:
self.bias = nn.Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 self.reset_parameters()
def reset_parameters(self)-> None:

 kaiming_uniform_for_vwl(self.weight, a=np.sqrt(5))
if self.bias is not None:
fan_in, _ = _calculate_fan_in_and_fan_out_for_vwl(self.weight)
 bound = 1 / np.sqrt(fan_in)
 nn.init.uniform_(self.bias, -bound, bound)
 def forward(self, X: torch.Tensor)-> torch.Tensor:

 H = self.matmul_func(X, self.weight)
 if self.bias is not None:
H = H + self.bias
return H
gc.collect()<concatenate>
class GraphConv(nn.Module):

 def __init__(
 self,
in_channels: int, out_channels: int, n_nodes: int, shrare_msg: bool=True,
model_self: bool=True, share_model_self: bool=True,
bias: bool=True, share_bias: bool=True 
)-> None:

 self.in_channels = in_channels
self.out_channels = out_channels
self.n_nodes = n_nodes
self.model_self = model_self
super(GraphConv, self ).__init__()
self.msg = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, shrare_msg)
 if model_self:
self.model_self = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, share_model_self)
 if bias:
if share_bias:
self.bias = nn.Parameter(torch.Tensor(out_channels))
else:
self.bias = nn.Parameter(torch.Tensor(n_nodes, out_channels))
bound = 1 / np.sqrt(out_channels)
 nn.init.uniform_(self.bias, -bound, bound)
 else:
self.register_parameter('bias', None)
 def forward(self, X: torch.Tensor, A: torch.Tensor, W: torch.Tensor=None)-> torch.Tensor:

 if W is not None:
A = A * W
M = X
M = self.msg(M)
 M = torch.matmul(A, M)
 H = M
if self.model_self:
H = H + self.model_self(X)
 if self.bias is not None:
H = H + self.bias
return H
X_float_features = np.hstack([X_item_descr_counts, X_cats_stats_features_scaled] )<string_transform>
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNStacking1d(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking1d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(
 in_ch, out_ch, kernel_size=3, stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class CNNStacking2d(nn.Module):

 def __init__(
 self, n_models: int, n_classes: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking2d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv2d(
 in_ch, out_ch, kernel_size=(1, 3), stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class GCNStacking(nn.Module):

 def __init__(
 self, n_classes: int,
n_channels_list: tp.List[int],
add_self_loop: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(GCNStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = GraphConv(
 in_ch, out_ch, n_classes,
shrare_msg=False, share_model_self=False, share_bias=False)
 setattr(self, "conv{}".format(i + 1), layer)
 self.relu = nn.ReLU(inplace=True)
 if add_self_loop:
adj_mat = torch.ones(n_classes, n_classes)/ n_classes
else:
adj_mat =(1 - torch.eye(n_classes, n_classes)) /(n_classes - 1)
 self.register_buffer("A", adj_mat.float())
 kwargs_head["n_features_list"][0] = n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, X: torch.FloatTensor)-> torch.Tensor:

 bs, n_classes = X.shape[:2]
H = X
for i in range(self.n_conv_layers):
H = getattr(self, "conv{}".format(i + 1))(H, self.A[None,...])
 H = self.relu(H)
 h = torch.reshape(H,(bs, -1))
h = self.head(h)
 return h
regex_tokenizer = RegexpTokenizer(r'[a-z][\w&]*|[\d]+[\.]*[\w]*|[/!?*:%$"'\-\+=\.,](?![/!?*:%$"'-\+=\.,])')
 def regex_tokenizer_nltk(text):
return regex_tokenizer.tokenize(text.lower() )<string_transform>
class EvalFuncManager(nn.Module):

 def __init__(
 self, iters_per_epoch: int,
evalfunc_dict: tp.Dict[str, nn.Module], prefix: str = "val" 
)-> None:

 super(EvalFuncManager, self ).__init__()
self.tmp_iter = 0
self.iters_per_epoch = iters_per_epoch
self.prefix = prefix
self.metric_names = []
for k, v in evalfunc_dict.items() :
setattr(self, k, v)
 self.metric_names.append(k)
 self.reset()
def reset(self)-> None:

 self.tmp_iter = 0
for name in self.metric_names:
getattr(self, name ).reset()
def __call__(self, y: torch.Tensor, t: torch.Tensor)-> None:

 for name in self.metric_names:
getattr(self, name ).update(y, t)
 self.tmp_iter += 1
if self.tmp_iter == self.iters_per_epoch:
ppe.reporting.report({
"{}/{}".format(self.prefix, name): getattr(self, name ).compute()
for name in self.metric_names})
 self.reset()
class MeanLoss(nn.Module):
def __init__(self):
super(MeanLoss, self ).__init__()
self.loss_sum = 0
self.n_examples = 0
def forward(self, y: torch.Tensor, t: torch.Tensor):

 return self.loss_func(y, t)
 def reset(self):

 self.loss_sum = 0
self.n_examples = 0
def update(self, y: torch.Tensor, t: torch.Tensor):

 self.loss_sum += self(y, t ).item() * y.shape[0]
self.n_examples += y.shape[0]
def compute(self):

 return self.loss_sum / self.n_examples
class MyLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLogLoss, self ).__init__()
self.loss_func = nn.BCEWithLogitsLoss(**params)
 class LSBCEWithLogitsLoss(nn.Module):

 def __init__(self, k: int, alpha: float=0.01):

 super(LSBCEWithLogitsLoss, self ).__init__()
self.k = k
self.alpha = alpha
self.loss_func = nn.BCEWithLogitsLoss()
def forward(self, y, t):

 t_s = t *(1 - self.alpha)+ self.alpha / self.k
loss = self.loss_func(y, t_s)
 return loss
class MyLSLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLSLogLoss, self ).__init__()
self.loss_func = LSBCEWithLogitsLoss(**params )
class Tokenizer:
def __init__(self, min_df=10, limit_length_transform=None, tokenizer=str.split, vocabulary=None,
unk_token_if_ootv=False, workers=1):
self.min_df = min_df
self.tokenizer = tokenizer
self.limit_length_transform = limit_length_transform
self.workers = workers
self.vocab_idx = vocabulary
self.unk_token_if_ootv = unk_token_if_ootv
self.max_len = None
def tokenize(self, texts):
if self.workers>1:
tokenized_texts = Parallel(n_jobs=self.workers )(delayed(self.tokenizer )(t)for t in texts)
 else:
tokenized_texts = [self.tokenizer(t)for t in texts]
return tokenized_texts
def fit(self, texts):
doc_freq = Counter()
max_len = 0
if type(texts)is list:
tokenized_texts = texts
else:
tokenized_texts = self.tokenize(texts)
 for sentence in tokenized_texts:
if self.vocab_idx == None:
doc_freq.update(set(sentence))
max_len = max(max_len, len(sentence))
self.max_len = max_len
if self.vocab_idx == None:
vocab = sorted([t for(t, c)in doc_freq.items() if c >= self.min_df])
 self.vocab_idx = {t:(i + 1)for(i, t)in enumerate(vocab)}
def fit_transform(self, texts):
self.fit(texts)
 return self.transform(texts)
 def text_to_idx(self, tokenized):
if self.unk_token_if_ootv:
return [self.vocab_idx[t] if t in self.vocab_idx else self.vocab_idx[UNK_TOKEN] for t in tokenized]
else:
return [self.vocab_idx[t] for t in tokenized if t in self.vocab_idx]
def transform(self, texts):
n = len(texts)
 max_length = self.limit_length_transform or self.max_len
result = np.zeros(shape=(n, max_length), dtype=np.int32)
 if self.workers>1:
tokenized_texts = Parallel(n_jobs=self.workers )(delayed(self.tokenizer )(t)for t in texts)
 else:
tokenized_texts = [self.tokenizer(t)for t in texts]
for i, sentence in enumerate(tokenized_texts):
text = self.text_to_idx(sentence[:max_length])
 result[i, :len(text)] = text
return result
def vocabulary_size(self):
return len(self.vocab_idx)+ 1<feature_engineering>
def run_train_loop(
 manager, args, model, device,
train_loader, optimizer, scheduler, loss_func 
):

 while not manager.stop_trigger:
model.train()
for batch in train_loader:
x, t = batch
with manager.run_iteration() :
optimizer.zero_grad()
y = model(x.to(device))
loss = loss_func(y, t.to(device))
ppe.reporting.report({'train/loss': loss.item() })
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
def run_eval(args, model, device, batch, eval_manager):

 model.eval()
x, t = batch
y = model(x.to(device))
eval_manager(y, t.to(device))
print('Generating cumulative sub-categories features...')
 def paths(tokens):
all_paths = ['/'.join(tokens[0:(i+1)])for i in range(len(tokens)) ]
return ' '.join(all_paths)
 whitespace_regex = re.compile(r'\s+')
 @lru_cache(1024)
 def cat_process(cat):
cat = cat.lower()
cat = whitespace_regex.sub('', cat)
 split = cat.split('/')
 return paths(split)
 df_train['category_name_cum'] = df_train.category_name.apply(cat_process)
 cat_tok = Tokenizer(min_df=50)
 cat_tok.fit(df_train.iloc[np.hstack([train_idxs, train_zeroed_prices_idxs])]['category_name_cum'])
 X_cat = cat_tok.transform(df_train['category_name_cum'])
 cat_voc_size = cat_tok.vocabulary_size()
print_elapsed()<train_model>
def get_optimizer(settings, model):
optimizer = getattr(torch.optim, settings["optimizer"]["name"] )(
 model.parameters() , **settings["optimizer"]["params"])
 return optimizer
def get_scheduler(settings, train_loader, optimizer):
if settings["scheduler"]["name"] is None:
scheduler = None
else:
if settings["scheduler"]["name"] == "OneCycleLR":
settings["scheduler"]["params"]["epochs"] = settings["globals"]["max_epoch"]
settings["scheduler"]["params"]["steps_per_epoch"] = len(train_loader)
 scheduler = getattr(torch.optim.lr_scheduler, settings["scheduler"]["name"] )(
 optimizer, **settings["scheduler"]["params"])
 return scheduler
def get_loss_function(settings):
if hasattr(nn, settings["loss"]["name"]):
loss_func = getattr(nn, settings["loss"]["name"] )(**settings["loss"]["params"])
 else:
loss_func = eval(settings["loss"]["name"] )(**settings["loss"]["params"])
 return loss_func
def get_manager(
 settings, model, device, train_loader, val_loader, optimizer, eval_manager, output_path 
):
trigger = ppe.training.triggers.EarlyStoppingTrigger(
 check_trigger=(1, 'epoch'), monitor='val/metric', mode="min",
patience=settings["globals"]["patience"], verbose=True,
max_trigger=(settings["globals"]["max_epoch"], 'epoch'))
manager = ppe.training.ExtensionsManager(
 model, optimizer, settings["globals"]["max_epoch"],
iters_per_epoch=len(train_loader), stop_trigger=trigger, out_dir=output_path)
 log_extentions = [
ppe_extensions.observe_lr(optimizer=optimizer),
ppe_extensions.LogReport() ,
ppe_extensions.PlotReport(["train/loss", "val/loss"], 'epoch', filename='loss.png'),
ppe_extensions.PlotReport(["lr"], 'epoch', filename='lr.png'),
ppe_extensions.PrintReport([
"epoch", "iteration", "lr", "train/loss", "val/loss", "val/metric", "elapsed_time"])
 ]
for ext in log_extentions:
manager.extend(ext)
 manager.extend(
 ppe_extensions.Evaluator(
 val_loader, model,
eval_func=lambda *batch: run_eval(settings, model, device, batch, eval_manager)) ,
trigger=(1, "epoch"))
manager.extend(
 ppe_extensions.snapshot(target=model, filename="snapshot_epoch_{.epoch}.pth"),
trigger=ppe.training.triggers.MinValueTrigger(key="val/metric", trigger=(1, 'epoch')))
 return manager
print('Processing vocabulary for name and description.... ')
 general_tokenizer = Tokenizer(min_df=30, tokenizer=regex_tokenizer_nltk, workers=4)
 general_tokenizer.fit(df_train.iloc[np.hstack([train_idxs, train_zeroed_prices_idxs])]['name'] + " " \
+ df_train.iloc[np.hstack([train_idxs, train_zeroed_prices_idxs])]['item_description'])
 print("Text vocabulary size: {}".format(len(general_tokenizer.vocab_idx)))
 print_elapsed()<string_transform>
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True
print('Creating vocabulary and loading/generating word embeddings...')
 words_vocab_with_embeddings = set(general_tokenizer.vocab_idx.keys() ).intersection(set(word_embedding_vocab.keys()))
words_vocab_no_embeddings = set(general_tokenizer.vocab_idx.keys())- words_vocab_with_embeddings
print('Found word embeddings for corpus: {} from {}'.format(len(words_vocab_with_embeddings), len(general_tokenizer.vocab_idx)))
 UNK_TOKEN = '<UNK>'
PAD_TOKEN = '<PAD>'
words_vocab_general = {t: i for(i, t)in enumerate([PAD_TOKEN, UNK_TOKEN] + sorted(words_vocab_no_embeddings)) }
words_without_embeddings_vocab_size = len(words_vocab_general)
 words_with_embeddings_vocab_size = len(words_vocab_with_embeddings)
 for(i, t)in enumerate(sorted(words_vocab_with_embeddings)) :
words_vocab_general[t] = i+ words_without_embeddings_vocab_size
custom_inv_vocab_words = dict([(idx,w)for w, idx in list(words_vocab_general.items())])
 total_vocab_size = len(words_vocab_general)
 print('Words without embedding: %d\tTotal vocabulary size: %d' %(words_without_embeddings_vocab_size, total_vocab_size))
embedding_size = word_embeddings_matrix.shape[1]
print("Embedding size: %d" %(embedding_size))
np.random.seed(10)
 max_abs_embedding_random_value = np.sqrt(2 / embedding_size)
 glove_scaling_factor = word_embeddings_matrix.max() / max_abs_embedding_random_value
words_with_embeddings_matrix = word_embeddings_matrix[[word_embedding_vocab[custom_inv_vocab_words[i]]
for i in range(words_without_embeddings_vocab_size, total_vocab_size)]] \
/ glove_scaling_factor
words_without_embeddings_matrix = np.random.normal(loc=words_with_embeddings_matrix.mean() ,
scale=words_with_embeddings_matrix.std() ,
size=(words_without_embeddings_vocab_size, embedding_size))
print("words_without_embeddings_matrix:", words_without_embeddings_matrix.shape)
 print("words_without_embeddings_matrix:", words_with_embeddings_matrix.shape)
 print_elapsed()<drop_column>
def run_train_one_fold(
 settings, model, train_all_dataset, train_val_index, device, output_path):

 train_dataset = data.Subset(train_all_dataset, train_val_index[0])
 val_dataset = data.Subset(train_all_dataset, train_val_index[1])
 train_loader = data.DataLoader(train_dataset, **settings["loader"]["train"])
 val_loader = data.DataLoader(val_dataset, **settings["loader"]["val"])
 print("train: {}, val: {}".format(len(train_dataset), len(val_dataset)))
 model.to(device)
 optimizer = get_optimizer(settings, model)
 scheduler = get_scheduler(settings, train_loader, optimizer)
 loss_func = get_loss_function(settings)
 loss_func.to(device)
 eval_mgr = EvalFuncManager(len(val_loader), {
"loss": loss_func,
"metric": MyLogLoss() ,
})
 manager = get_manager(
 settings, model, device, train_loader, val_loader,
optimizer, eval_mgr, output_path)
 run_train_loop(
 manager, settings, model, device, train_loader, optimizer, scheduler, loss_func )
del(word_embeddings_matrix )<define_variables>
def get_drug_multilabel_index(target_in_drug_df, target_cols_list, vc_num, n_folds, seed):

 vc = target_in_drug_df.drug_id.value_counts()
vc1 = vc.loc[vc <= vc_num].index
vc2 = vc.loc[vc > vc_num].index
dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.groupby('drug_id')[target_cols_list].mean().loc[vc1]
if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.loc[target_in_drug_df.drug_id.isin(vc2)].reset_index(drop = True)
 if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 folds = []
target_in_drug_df['fold'] = target_in_drug_df.drug_id.map(dct1)
 target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'fold'] = target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'sig_id'].map(dct2)
 target_in_drug_df.fold = target_in_drug_df.fold.astype('int8')
 folds.append(target_in_drug_df.fold.values)
 return np.stack(folds)[0]
def drug_multilabel_stratified_kfold(k, target_in_drug_df, target_cols_list, n_folds, seed):

 folds_array = get_drug_multilabel_index(target_in_drug_df, target_cols_list, 18, n_folds, seed)
 train_idx = [i for i, x in enumerate(folds_array)if x != k]
validation_idx = [i for i, x in enumerate(folds_array)if x == k]
return train_idx, validation_idx
NAME_MAX_LEN = 20
ITEM_DESCR_MAX_LEN = 70<train_model>
target_in_drug = pd.merge(
 pd.read_csv(".. /input/lish-moa/train_targets_scored.csv"),
pd.read_csv(".. /input/lish-moa/train_drug.csv"),
how='left', on='sig_id')
 print(target_in_drug.shape)
 target_in_drug_trt = del_control(target_in_drug)
 print(target_in_drug_trt.shape )
print('Processing Title...')
 name_tok = Tokenizer(min_df=0, limit_length_transform=NAME_MAX_LEN, tokenizer=regex_tokenizer_nltk,
vocabulary=words_vocab_general, unk_token_if_ootv=True, workers=4)
 name_tok.fit(df_train.iloc[np.hstack([train_idxs, train_zeroed_prices_idxs])]['name'])
 X_name = name_tok.transform(df_train.name)
 print(X_name.shape)
 print_elapsed()<categorify>
X = np.concatenate([df.values for df in oof_list], axis=1)
 print(X.shape)
 y = y_true.values
print(y.shape)
 X_2d = np.stack([df.values for df in oof_list], axis=2)
 print(X_2d.shape)
 X_3d = X_2d[:, None,...]
print(X_3d.shape)
 X_node = np.stack([df.values for df in oof_list], axis=2 )
print('Processing Description...')
 desc_tok = Tokenizer(min_df=0, limit_length_transform=ITEM_DESCR_MAX_LEN, tokenizer=regex_tokenizer_nltk,
vocabulary=words_vocab_general, unk_token_if_ootv=True, workers=4)
 desc_tok.fit(df_train.iloc[np.hstack([train_idxs, train_zeroed_prices_idxs])].item_description)
 X_desc = desc_tok.transform(df_train.item_description)
 print(X_desc.shape)
 print_elapsed()<data_type_conversions>
train_all_dataset = MoAStackingDatasetForGCN(X_node, y )
print('Processing Item condition and Shipping...')
 X_item_cond =(df_train.item_condition_id - 1 ).astype('uint8' ).values.reshape(-1, 1)
 X_shipping = df_train.shipping.astype('float32' ).values.reshape(-1, 1)
 print_elapsed()<define_variables>
TRAIN_OUTPUT = Path.cwd().parent / "tmp"
TRAIN_OUTPUT.mkdir(exist_ok=True)
 WORK_DIR = Path.cwd()
def prepare_batches(seq, step):
n = len(seq)
 res = []
for i in range(0, n, step):
res.append(seq[i:i+step])
 return res<train_model>
gc.collect()
def train_model(session, epochs=4, batch_size=500, eval_each_epoch=True, dropout_input_words=0.0):
print('
training the model...')
 print_elapsed()
training_size = train_idxs.shape[0]
for i in range(int(np.ceil(epochs))):
print("-----------------EPOCH: {}-------------------".format(i))
t0 = time()
np.random.seed(i)
 epoch_size = training_size
if i+1 - epochs > 0:
epoch_size = int(training_size*(epochs%1))
batches = prepare_batches(np.random.permutation(train_idxs)[:epoch_size], batch_size)
 for idx in batches:
name_batch = X_name[idx]
desc_batch = X_desc[idx]
if dropout_input_words > 0:
name_mask =(np.random.uniform(0,1, size=name_batch.shape)>= dropout_input_words ).astype(np.int32)
 name_batch = name_batch * name_mask
desc_mask =(np.random.uniform(0,1, size=desc_batch.shape)>= dropout_input_words ).astype(np.int32)
 desc_batch = desc_batch * desc_mask
feed_dict = {
place_name: name_batch,
place_desc: desc_batch,
place_brand: X_brand[idx],
place_cat: X_cat[idx],
place_cond: X_item_cond[idx],
place_ship: X_shipping[idx],
place_float_stats: X_float_features[idx],
place_y: y[idx],
place_training: True
}
session.run(train_step, feed_dict=feed_dict)
 took = time() - t0
print('epoch %d took %.3fs' %(i, took))
if eval_each_epoch and i < epochs-1:
train_set_evaluation(sess)
 print_elapsed()
eval_set_evaluation(session)
 print_elapsed()<split>
settings_str =
settings = yaml.safe_load(settings_str )
def eval_set_evaluation(session, save_results_to=None):
print('
EVAL SET Evaluation')
 y_pred_norm_eval = np.zeros(valid_size)
 EVAL_BATCH_SIZE = 5000
batches = prepare_batches(valid_idxs, EVAL_BATCH_SIZE)
 for b, idx in enumerate(batches):
feed_dict = {
place_name: X_name[idx],
place_desc: X_desc[idx],
place_brand: X_brand[idx],
place_cat: X_cat[idx],
place_cond: X_item_cond[idx],
place_ship: X_shipping[idx],
place_float_stats: X_float_features[idx],
place_training: False
}
batch_pred = session.run(out, feed_dict=feed_dict)
 start_idx =(b*EVAL_BATCH_SIZE)
 y_pred_norm_eval[start_idx:min(start_idx+EVAL_BATCH_SIZE, valid_size)] = batch_pred[:, 0]
y_adjusted_pred_eval = np.expm1(y_pred_norm_eval * price_log_train_std + price_log_train_mean)
 print("Eval set RMSLE: {}".format(rmsle(price[valid_idxs].values, y_adjusted_pred_eval)))
 if not SUBMISSION and save_results_to != None:
print('Saving CNN eval results to "valid_cnn_predictions.csv"...')
 df_out = pd.DataFrame()
df_out['train_id'] = df_train.iloc[valid_idxs]['train_id']
df_out['price'] = price[valid_idxs].values
df_out['pred_price'] = y_adjusted_pred_eval
df_out.to_csv(save_results_to, index=False)
 return y_pred_norm_eval<prepare_x_and_y>
device = torch.device(settings["globals"]["device"])
 cv_score_df_list = []
if settings["globals"]["fast_commit"]:
settings["globals"]["max_epoch"] = 1
stgs_list = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list.append(deepcopy(settings))
stgs_list[-1]["globals"]["seed"] = tmp_seed
def train_set_evaluation(session):
print('
TRAIN SET Evaluation')
 y_pred_norm = np.zeros(train_size)
 EVAL_BATCH_SIZE = 5000
batches = prepare_batches(train_idxs, EVAL_BATCH_SIZE)
 for b, idx in enumerate(batches):
feed_dict = {
place_name: X_name[idx],
place_desc: X_desc[idx],
place_brand: X_brand[idx],
place_cat: X_cat[idx],
place_cond: X_item_cond[idx],
place_ship: X_shipping[idx],
place_float_stats: X_float_features[idx],
place_training: False
}
batch_pred = session.run(out, feed_dict=feed_dict)
 start_idx =(b*EVAL_BATCH_SIZE)
 y_pred_norm[start_idx:min(start_idx+EVAL_BATCH_SIZE, train_size)] = batch_pred[:, 0]
y_pred_train = y_pred_norm * price_log_train_std + price_log_train_mean
y_adjusted_pred_train = np.expm1(y_pred_train)
 print("Train set RMSLE: {}".format(rmsle(price[train_idxs].values, y_adjusted_pred_train)))
 return price[train_idxs].values, y_adjusted_pred_train<save_to_csv>
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = settings["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
set_random_seed(tmp_seed, True)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 run_train_one_fold(
 tmp_stgs, model, train_all_dataset, train_val_indexs[fold_id], device,
TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id))
del model
gc.collect()
best_eval_list = []
for fold_id in range(n_folds):
output_dir = TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id)
 log_df = pd.read_json(output_dir / "log")
 best_log = log_df.iloc[log_df["val/metric"].idxmin() ][["epoch", "train/loss", "val/loss", "val/metric"]]
best_eval_list.append(best_log)
 best_epoch = int(best_log["epoch"])
 model_path = output_dir / "snapshot_epoch_{}.pth".format(best_epoch)
 cp_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 shutil.copy(model_path, cp_path)
 for model_path in output_dir.glob("*.pth"):
model_path.unlink()
cv_score_df = pd.DataFrame(best_eval_list)
 cv_score_df.insert(0, "fold", range(settings["globals"]["n_folds_split"]))
cv_score_df.insert(0, "model", m_id)
 del best_eval_list
cv_score_df_list.append(cv_score_df )
def save_test_set_predictions(preds, filename):
df_out = pd.DataFrame()
df_out['test_id'] = range(0,preds.shape[0])
 df_out['price'] = preds
df_out.to_csv(filename, index=False)
 print_elapsed('Predictions exported to {}'.format(filename))<load_from_csv>
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )
def generate_test_submission(session):
print('Reading and generating features for test data...')
 df_test = pd.read_csv('.. /input/mercari-price-suggestion-challenge/test_stg2.tsv', sep='\t')
 df_test.name.fillna('unk_name', inplace=True)
 df_test.category_name.fillna('unk_cat', inplace=True)
 df_test.brand_name.fillna('unk_brand', inplace=True)
 df_test.item_description.fillna('unk_brand', inplace=True)
 test_names_unknown_brands = df_test[df_test['brand_name'] == 'unk_brand'][['name','category_name']].astype('str' ).values
test_estimated_brands = Parallel(n_jobs=7 )(delayed(brandfinder )(name, category)for name, category in test_names_unknown_brands)
 df_test.ix[df_test['brand_name'] == 'unk_brand', 'brand_name'] = test_estimated_brands
df_test.category_name = df_test.category_name.apply(cat_process)
 df_test.brand_name = df_test.brand_name.str.lower()
df_test.brand_name = df_test.brand_name.str.replace(' ', '_')
 X_item_descr_counts_test = item_descr_counts_scaler.transform(np.vstack(df_test['item_description'].astype(str)\ 
.apply(extract_counts ).values))
X_cats_stats_features_scaled_test = cats_stats_features_scaler.transform(merge_with_cat_stats(df_test))
X_float_features_test = np.hstack([X_item_descr_counts_test, X_cats_stats_features_scaled_test])
 X_cat_test = cat_tok.transform(df_test.category_name)
 X_name_test = name_tok.transform(df_test.name)
 X_desc_test = desc_tok.transform(df_test.item_description)
 X_item_cond_test =(df_test.item_condition_id - 1 ).astype('uint8' ).values.reshape(-1, 1)
 X_shipping_test = df_test.shipping.astype('float32' ).values.reshape(-1, 1)
 X_brand_test = df_test.brand_name.apply(lambda b: brands_idx.get(b, 0))
X_brand_test = X_brand_test.values.reshape(-1, 1)
 print_elapsed('Finish generating features for test set')
 print('Prediction for test set using the trained CNN model...')
 n_test = len(df_test)
 y_pred = np.zeros(n_test)
 test_idx = np.arange(n_test)
 batches = prepare_batches(test_idx, 5000)
 for idx in batches:
feed_dict = {
place_name: X_name_test[idx],
place_desc: X_desc_test[idx],
place_brand: X_brand_test[idx],
place_cat: X_cat_test[idx],
place_cond: X_item_cond_test[idx],
place_ship: X_shipping_test[idx],
place_float_stats: X_float_features_test[idx],
place_training: False
}
batch_pred = session.run(out, feed_dict=feed_dict)
 y_pred[idx] = batch_pred[:, 0]
print_elapsed()
return y_pred<set_options>
if DO_SUBMISSION or DO_VIRTUAL_SUBMISSION:
sub_list = [
pd.read_csv('./submission-sinchir0-nn.csv'),
pd.read_csv('./submission_sinchir0_tabnet.csv'),
pd.read_csv("./submission_takapy_tf-resnet.csv"),
pd.read_csv('./submission_tawara_thrnn_seed_cv.csv'),
pd.read_csv('./submission_tawara_thrnn_drug_seed_cv.csv'),
]
else:
sub_list = [
pd.read_csv('.. /input/nn-use-train-public/submission.csv'),
pd.read_csv('.. /input/tabnet-train-public-add-n-shared-1/submission.csv'),
pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/submission.csv"),
pd.read_csv('.. /input/moa-weight-thrnn-seed-cv/submission.csv'),
pd.read_csv('.. /input/moa-weight-thrnn-drug-seed-cv/submission.csv'),
]
for i, name in enumerate(["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]):
print(f"[{name}]:", sub_list[i].shape )
gc.collect()<categorify>
sub_list = [order_sub(sub_df)for sub_df in sub_list]
print('Defining the model...')
 def conv1d(inputs, num_filters, filter_size, pool_size, is_training, reg=0.0, activation=None, padding='same'):
out = tf.layers.conv1d(
 inputs=inputs, filters=num_filters, padding=padding,
kernel_size=filter_size,
strides=1,
activation=activation,
kernel_initializer=tf.contrib.layers.xavier_initializer() ,
kernel_regularizer=tf.contrib.layers.l2_regularizer(reg), 
)
 out = tf.layers.max_pooling1d(out, pool_size=pool_size, strides=1, padding='valid')
 return out
def dense(X, size, reg=0.0, activation=None):
out = tf.layers.dense(X, units=size, activation=activation,
kernel_initializer=tf.contrib.layers.xavier_initializer() ,
kernel_regularizer=tf.contrib.layers.l2_regularizer(reg))
return out
def embed(inputs, size, dim):
std = np.sqrt(2 / dim)
 emb = tf.Variable(tf.random_uniform([size, dim], -std, std))
lookup = tf.nn.embedding_lookup(emb, inputs)
 return lookup
dnn_settings = {
'lr_initial': 0.002,
'lr_decay_rate': 0.94,
'lr_num_epochs_per_decay': 0.1,
'batch_size': 256,
'epochs': 3.0,
'dropout_input_words': 0.05,
'dropout_rate': 0.00,
'l2_reg': 0.0,
'main_batch_norm_decay': 0.93,
'cnn_filter_sizes': [3],
'name_seq_len': NAME_MAX_LEN,
'name_num_filters': 128,
'name_avg_embedding_num_words': 5,
'desc_seq_len': ITEM_DESCR_MAX_LEN,
'desc_num_filters': 96,
'descr_avg_embedding_num_words': 20,
'brand_embeddings_dim': 32,
'cat_embeddings_dim': 32,
'cat_seq_len': X_cat.shape[1],
'word_embeddings_size': embedding_size,
'word_embeddings_max_norm': 0.45,
}
print("SETTINGS:", dnn_settings)
 print()
graph = tf.Graph()
graph.seed = 1
ALLOW_SOFT_PLACEMENT=True
LOG_DEVICE_PLACEMENT=False
with graph.as_default() :
with tf.device('/cpu:0' if SUBMISSION else '/gpu:0'):
session_conf = tf.ConfigProto(
 allow_soft_placement=ALLOW_SOFT_PLACEMENT,
log_device_placement=LOG_DEVICE_PLACEMENT,
intra_op_parallelism_threads = 4,
inter_op_parallelism_threads = 4)
 sess = tf.Session(config=session_conf)
 with sess.as_default() :
with tf.name_scope("embedding"):
words_with_embedding_placeholder = tf.placeholder(tf.float32, [words_with_embeddings_vocab_size,
dnn_settings['word_embeddings_size']])
 words_with_embedding_variable = tf.Variable(tf.constant(0.0, shape=[words_with_embeddings_vocab_size,
dnn_settings['word_embeddings_size']]),
trainable=True,
name="words_with_embedding")
 words_with_embedding_init = words_with_embedding_variable.assign(words_with_embedding_placeholder)
 words_without_embedding_placeholder = tf.placeholder(tf.float32, [words_without_embeddings_vocab_size,
dnn_settings['word_embeddings_size']])
 words_without_embedding_variable = tf.Variable(tf.constant(0.0, shape=[words_without_embeddings_vocab_size,
dnn_settings['word_embeddings_size']]),
trainable=True,
name="words_without_embedding")
 words_without_embedding_init = words_without_embedding_variable.assign(words_without_embedding_placeholder)
 word_embedding_regularizer = tf.nn.l2_loss(words_without_embedding_variable)
 words_embedding_variable = tf.concat([words_without_embedding_variable, words_with_embedding_variable],
axis=0)
 place_name = tf.placeholder(tf.int32, shape=(None, dnn_settings['name_seq_len']))
place_desc = tf.placeholder(tf.int32, shape=(None, dnn_settings['desc_seq_len']))
place_brand = tf.placeholder(tf.int32, shape=(None, 1))
place_cat = tf.placeholder(tf.int32, shape=(None, dnn_settings['cat_seq_len']))
place_ship = tf.placeholder(tf.float32, shape=(None, 1))
place_cond = tf.placeholder(tf.uint8, shape=(None, 1))
place_float_stats = tf.placeholder(dtype=tf.float32, shape=(None, X_float_features.shape[1]))
place_y = tf.placeholder(dtype=tf.float32, shape=(None, 1))
place_training = tf.placeholder(tf.bool, shape=() ,)
 brand = embed(place_brand, brand_voc_size, dnn_settings['brand_embeddings_dim'])
 cat = embed(place_cat, cat_voc_size, dnn_settings['cat_embeddings_dim'])
 name = tf.nn.embedding_lookup(words_embedding_variable, place_name, max_norm=dnn_settings['word_embeddings_max_norm'])
 desc = tf.nn.embedding_lookup(words_embedding_variable, place_desc, max_norm=dnn_settings['word_embeddings_max_norm'])
 print("name.shape", name.shape)
 print("desc.shape", desc.shape)
 name_mean_embeddings = tf.reduce_mean(name[:,:dnn_settings['name_avg_embedding_num_words'],:], axis=1)
 print("name_mean_embeddings.shape", name_mean_embeddings.shape)
 desc_mean_embeddings = tf.reduce_mean(desc[:,:dnn_settings['descr_avg_embedding_num_words'],:], axis=1)
 print("desc_mean_embeddings.shape", desc_mean_embeddings.shape)
 conv_layers = []
for filter_size in dnn_settings['cnn_filter_sizes']:
with tf.name_scope("conv-maxpool-%s" % filter_size):
name_conv = conv1d(name, num_filters=dnn_settings['name_num_filters'], filter_size=filter_size,
pool_size=dnn_settings['name_seq_len'],
is_training=place_training, reg=dnn_settings['l2_reg'], activation=tf.nn.relu)
 name_conv = tf.contrib.layers.flatten(name_conv)
 print(( "conv-maxpool-%s NAME" % filter_size), name_conv.shape)
 conv_layers.append(name_conv)
 desc_conv = conv1d(desc, num_filters=dnn_settings['desc_num_filters'], filter_size=filter_size,
pool_size=dnn_settings['desc_seq_len'],
is_training=place_training, reg=dnn_settings['l2_reg'], activation=tf.nn.relu)
 desc_conv = tf.contrib.layers.flatten(desc_conv)
 print(( "conv-maxpool-%s DESCRIPTION" % filter_size), desc_conv.shape)
 conv_layers.append(desc_conv)
 brand = tf.contrib.layers.flatten(brand)
 print(brand.shape)
 cat = tf.contrib.layers.flatten(cat)
 print(cat.shape)
 ship = place_ship
print(ship.shape)
 cond = tf.one_hot(place_cond, 5)
 cond = tf.contrib.layers.flatten(cond)
 print(cond.shape)
 float_stats = place_float_stats
float_stats = tf.contrib.layers.flatten(float_stats)
 print(float_stats.shape)
 out = tf.concat(conv_layers + [name_mean_embeddings, desc_mean_embeddings, brand, cat, ship, cond, float_stats], axis=1)
 print('concatenated dim:', out.shape)
 out = tf.contrib.layers.batch_norm(out, decay=dnn_settings['main_batch_norm_decay'],
center=True, scale=False, epsilon=0.001,
is_training=place_training)
 out = dense(out, 256, reg=dnn_settings['l2_reg'], activation=tf.nn.relu)
 out = dense(out, 128, reg=dnn_settings['l2_reg'], activation=tf.nn.relu)
 out = tf.contrib.layers.layer_norm(tf.concat([out, ship, cond, float_stats], axis=1))
out = dense(out, 1, reg=dnn_settings['l2_reg'])
 reg_loss = tf.losses.get_regularization_loss() + dnn_settings['l2_reg']*word_embedding_regularizer
loss = tf.losses.mean_squared_error(place_y, out)+ reg_loss
rmse = tf.sqrt(loss)
 lr_decay_steps = int(train_size / dnn_settings['batch_size'] * dnn_settings['lr_num_epochs_per_decay'])
 global_step = tf.Variable(0, trainable=False)
 learning_rate_decay = tf.train.exponential_decay(dnn_settings['lr_initial'],
global_step,
lr_decay_steps,
dnn_settings['lr_decay_rate'],
staircase=True,
name='exponential_decay_learning_rate')
 opt = tf.train.AdamOptimizer(learning_rate=learning_rate_decay,
beta1=0.9,
beta2=0.999,
epsilon=1e-08 
)
 update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
 with tf.control_dependencies(update_ops):
train_step = opt.minimize(loss=loss, global_step=global_step)
 print("Initializing variables")
 init = tf.global_variables_initializer()
sess.run(init)
 print_elapsed()
sess.run([words_without_embedding_init,
words_with_embedding_init],
feed_dict={words_without_embedding_placeholder: words_without_embeddings_matrix,
words_with_embedding_placeholder: words_with_embeddings_matrix})
 train_model(sess, epochs=dnn_settings['epochs'], batch_size=dnn_settings['batch_size'],
dropout_input_words=dnn_settings['dropout_input_words'],
eval_each_epoch=not SUBMISSION)
 print_elapsed()
if not SUBMISSION:
train_actual_prices_debug, train_pred_prices_debug = train_set_evaluation(sess)
 print_elapsed()
cnn_pred_eval = eval_set_evaluation(sess)
 print_elapsed()
if SUBMISSION:
cnn_pred_test = generate_test_submission(sess)
 cnn_pred_test_scaled = np.expm1(cnn_pred_test * price_log_train_std + price_log_train_mean)
 save_test_set_predictions(cnn_pred_test_scaled, 'submission_cnn.csv' )<set_options>
def inference_function(settings, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in loader:
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr
%matplotlib inline<import_modules>
X_test = np.concatenate([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=1)
 print(X_test.shape)
 X_test_2d = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 print(X_test_2d.shape)
 X_test_3d = X_test_2d[:, None,...]
print(X_test_3d.shape)
 X_test_node = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 test_dataset = MoAStackingDatasetForGCN(X_test_node, None )
import numpy as np
import pandas as pd
from PIL import Image
import matplotlib.pyplot as plt
import os
import cv2
import glob
import tensorflow as tf
import keras
from keras.models import load_model, save_model
from keras.layers import Input, GlobalAveragePooling2D, GlobalMaxPooling2D
import keras.backend as K
from keras.models import Model, load_model
from keras.applications import VGG16
from keras.applications.vgg16 import preprocess_input<load_from_csv>
oof_pred_arr_avg = np.zeros(( len(X), 206))
test_pred_arr_avg = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 test_dataset.model_order = train_all_dataset.model_order
print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 model = GCNStacking(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg += oof_pred_arr
test_pred_arr_avg += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_df_list[m_id] = cv_score_df_list[m_id].append(
 {"model": m_id, "fold": "oof", "val/metric": oof_score,},
ignore_index=True)
 oof_pred_arr_avg /= len(stgs_list)
 test_pred_arr_avg /= len(stgs_list )
train_df = pd.read_csv('.. /input/landmark-retrieval-2020/train.csv')
 train_df.head()<filter>
cv_score_all = pd.concat(cv_score_df_list, axis=0)
 cv_score_all
train_df[train_df.id == '000171b259e48280']<filter>
print(log_loss(y.ravel() , oof_pred_arr_avg.ravel()))
train_df[train_df.landmark_id == 5724]<count_values>
if DO_SUBMISSION or DO_TRAIN_FOR_ENSEMBLE:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 elif DO_VIRTUAL_SUBMISSION:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 print(df_test.shape, submission.shape)
 df_test, submission = generate_virtual_private(df_test, submission)
 print(df_test.shape, submission.shape)
 else:
raise ValueError
<define_variables><EOS> 
df = submission.copy()
public_id = list(df['sig_id'].values)
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type =='ctl_vehicle'].sig_id] = 0
df_submit.to_csv('submission.csv',index=True )