Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<load_from_csv>
!pip install.. /input/sacremoses/sacremoses-master/ > /dev/null
!pip install.. /input/transformers/transformers-2.3.0/ > /dev/null
!pip install.. /input/iterative-stratification/iterative-stratification-master/ > /dev/null
df1 = pd.read_csv('.. /input/notebookf0a11d076c/submission.csv')
 df2 = pd.read_csv('.. /input/notebook309ee3e26d/submission.csv')
 df3 = pd.read_csv('.. /input/notebooka050851501/submission.csv' )<sort_values>
import re
import random
import logging
from typing import List, Dict, Tuple, Any
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<sort_values>
import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, MinMaxScaler
from sklearn.model_selection import GroupKFold
import joblib
import tensorflow as tf
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Sequential, Model, load_model
from tensorflow.keras.layers import Dense, Activation, Dropout, Input, GlobalAveragePooling1D, Lambda
from tensorflow.keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
import tensorflow.keras.backend as K
import tensorflow_hub
import transformers
from transformers import TFBertModel, BertTokenizer
from scipy.stats import spearmanr
from tqdm import tqdm
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import MultiTaskElasticNet
df1 = order_sub(df1)
 df2 = order_sub(df2)
 df3 = order_sub(df3 )<prepare_output>
pd.options.display.max_columns = None
tf.get_logger().setLevel(logging.ERROR)
 random.seed(31)
 np.random.seed(31)
 tf.random.set_seed(31 )
BLEND=df1.copy()
BLEND.iloc[:,1:] = df1.iloc[:,1:]*0.40 +df2.iloc[:,1:]*0.45 +df3.iloc[:,1:]*0.15<define_variables>
train = pd.read_csv('.. /input/google-quest-challenge/train.csv')
 target_cols = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv' ).columns[1:].tolist()
features_train, targets_train = train.drop(columns=target_cols), train.loc[:, target_cols]
del train
features_test = pd.read_csv('.. /input/google-quest-challenge/test.csv' )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL )<load_from_csv>
class BaseTransformer(BaseEstimator, TransformerMixin):
def fit(self, x: pd.DataFrame, y = None):
return self
def transform(self, x: pd.DataFrame)-> pd.DataFrame:
return x
class ColumnTransformer(BaseTransformer):
def __init__(self, defs: Dict[str, BaseTransformer]):
self.defs = defs
def fit(self, x: pd.DataFrame, y: np.ndarray = None):
for col, transformer in self.defs.items() :
transformer.fit(x[col], y)
 return self
def transform(self, x: pd.DataFrame)-> pd.DataFrame:
xp = x.copy()
for col, transformer in self.defs.items() :
xp[col] = transformer.transform(x[col])
 return xp
def fit_transform(self, x: pd.DataFrame, y: np.ndarray = None)-> pd.DataFrame:
xp = x.copy()
for col, transformer in self.defs.items() :
if hasattr(transformer, 'fit_transform'):
xp[col] = transformer.fit_transform(x[col], y)
 else:
xp[col] = transformer.fit(x[col], y ).transform(x[col])
 return xp
class WrappedOneHotEncoder(BaseTransformer):
def __init__(self, col: str):
self.col = col
self.oe = OneHotEncoder(drop='first', sparse=False)
 def fit(self, x: pd.DataFrame, y = None):
self.oe.fit(x.loc[:, [self.col]])
 return self
def transform(self, x: pd.DataFrame)-> pd.DataFrame:
sparse_matrix = self.oe.transform(x.loc[:, [self.col]])
 columns = ['{0}_onehot_{1}'.format(self.col, i)for i in range(sparse_matrix.shape[1])]
onehot = pd.DataFrame(sparse_matrix, index=x.index, columns=columns)
 return pd.concat([x, onehot], axis=1)
 class WrappedMinMaxScaler(BaseTransformer):
def __init__(self, minmax: Tuple[float, float] =(1e-5, 1-1e-5)) :
self.mms = MinMaxScaler(minmax)
 def fit_transform(self, x: pd.DataFrame)-> pd.DataFrame:
array = self.mms.fit_transform(x)
 return pd.DataFrame(array, index=x.index, columns=x.columns )
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<set_options>
def colwise_spearmanr(y_true: np.ndarray, y_pred: np.ndarray, cols: List[str])-> Dict[str, float]:
return {c: spearmanr(y_true[:, i], y_pred[:, i] ).correlation for i, c in enumerate(cols)}
def average_spearmanr(y_true: np.ndarray, y_pred: np.ndarray)-> float:
return np.average([
spearmanr(y_t, y_p ).correlation for y_t, y_p in zip(y_true.T, y_pred.T)
 ])
 def mini_batch(l: List, batch_size):
for i in range(0, len(l), batch_size):
yield l[i:i + batch_size]
def rank_average(arrays: List[np.ndarray])-> np.ndarray:
rank_sum = np.sum([pd.DataFrame(a ).rank().values for a in arrays], axis=0)
 return rank_sum /(len(arrays)* arrays[0].shape[0])
 def pd_average(df_ys: List[pd.DataFrame])-> pd.DataFrame:
return pd.DataFrame(
 np.average([df_y.values for df_y in df_ys], axis=0),
columns=df_ys[0].columns,
index=df_ys[0].index, 
)
 def pd_rank_average(df_ys: List[pd.DataFrame])-> pd.DataFrame:
return pd.DataFrame(
 rank_average([df_y.values for df_y in df_ys]),
columns=df_ys[0].columns,
index=df_ys[0].index,
)
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
<load_from_csv>
class SecondLevelDomainExtracter(BaseTransformer):
def transform(self, s_in: pd.Series)-> pd.Series:
s = s_in.str.extract(r'(^|.*\.)([^\.]+\.[^\.]+$)' ).iloc[:, 1]
s.name = s_in.name
return s
class UniversalSentenceEncoderEncoder(BaseTransformer):
def __init__(self, col: str, model, batch_size: int = 16):
self.col = col
self.model = model
self.batch_size = batch_size
def transform(self, df: pd.DataFrame)-> pd.DataFrame:
texts = df.loc[:, self.col].str.replace('?', '.' ).str.replace('!', '.' ).values
pbar_total = len(df)// self.batch_size + 1
pbar_desc = '{0} -> USE'.format(self.col)
 use_features = np.vstack([
self.model(texts_batch)['outputs'].numpy() for texts_batch in tqdm(
 mini_batch(texts, self.batch_size),
total=pbar_total,
desc=pbar_desc 
)
 ])
 columns = ['{0}_use{1}'.format(self.col, i)for i in range(use_features.shape[1])]
return pd.concat([df, pd.DataFrame(use_features, index=df.index, columns=columns)], axis=1)
 class DistanceEngineerer(BaseTransformer):
def __init__(self, col1: str, col2: str):
self.col1 = col1
self.col2 = col2
@staticmethod
def extract_matrix(df: pd.DataFrame, base_col: str)-> np.ndarray:
columns = [c for c in df.columns if re.fullmatch(base_col + r'\d+', c)is not None]
return df.loc[:, columns].values
def transform(self, df_in: pd.DataFrame)-> pd.DataFrame:
df = df_in.copy()
a1 = self.extract_matrix(df, self.col1)
 a2 = self.extract_matrix(df, self.col2)
 assert a1.shape == a2.shape
l2_distance = np.power(a1 - a2, 2 ).sum(axis=1)
 cos_distance =(a1 * a2 ).sum(axis=1)
 df.loc[:, 'l2dist_{0}-{1}'.format(self.col1, self.col2)] = l2_distance
df.loc[:, 'cosdist_{0}-{1}'.format(self.col1, self.col2)] = cos_distance
return df
class QATokenizer(BaseTransformer):
def __init__(self, tokenizer, max_len: int = 512):
self.tokenizer = tokenizer
self.max_len = max_len
def tokenize(self, question_title: str, question_body: str, answer: str)-> Tuple[List[int], List[int], List[int]]:
question = '{0}[SEP]{1}'.format(question_title, question_body)
 return self.tokenizer.encode_plus(question, answer, max_length=self.max_len, pad_to_max_length=True)
 def transform(self, df_in: pd.DataFrame)-> pd.DataFrame:
tokenized = [self.tokenize(
 row['question_title'],
row['question_body'],
row['answer'] 
)for _, row in df_in.iterrows() ]
input_ids = [d['input_ids'] for d in tokenized]
attention_mask = [d['attention_mask'] for d in tokenized]
token_type_ids = [d['token_type_ids'] for d in tokenized]
return pd.DataFrame(np.hstack([input_ids, attention_mask, token_type_ids]), index=df_in.index)
 class ColumnDropper(BaseTransformer):
def __init__(self, cols: List[str]):
self.cols = cols
def transform(self, df_in: pd.DataFrame)-> pd.DataFrame:
return df_in.drop(columns=self.cols )
train=pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_target = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv" )<load_from_csv>
class BaseEnsembleCV:
def __init__(self, n_splits: int = 5, verbose: bool = False, name: str = 'base_ecv'):
self.n_splits = n_splits
self.verbose = verbose
self.name = name
def fit_fold(self, x_train, y_train, x_val, y_val, **params):
raise NotImplementedError()
def save_fold(self, model, i_fold, dist):
raise NotImplementedError()
def load_fold(self, i_fold, src):
raise NotImplementedError()
def fit(self, df_x: pd.DataFrame, df_y: pd.DataFrame, groups = None, **params):
x = df_x.values
y = df_y.values
if groups is None:
kfold = MultilabelStratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=31 ).split(x, y)
 else:
kfold = GroupKFold(n_splits=self.n_splits ).split(x, y, groups=groups)
 iterator = enumerate(kfold)
 if self.verbose:
iterator = tqdm(iterator, total=self.n_splits, desc='k-fold')
 self.models = []
_y_oof = np.zeros(y.shape)
 for i_fold,(i_train, i_val)in iterator:
x_train, x_val = x[i_train], x[i_val]
y_train, y_val = y[i_train], y[i_val]
model = self.fit_fold(x_train, y_train, x_val, y_val, i_fold, **params)
 self.models.append(model)
 _y_oof[i_val] = model.predict(x_val)
 _score = colwise_spearmanr(y, _y_oof, df_y.columns)
 self.y_oof = pd.DataFrame(_y_oof, index=df_y.index, columns=df_y.columns)
 self.score = pd.Series(_score)
 self.save()
return self
def predict(self, df_x: pd.DataFrame)-> pd.DataFrame:
x = df_x.values
y = np.average([
m.predict(x)for m in self.models
], axis=0)
 return pd.DataFrame(y, index=df_x.index, columns=target_cols)
 def save(self, dist: str = '.'):
for i_fold, m in enumerate(self.models):
self.save_fold(m, i_fold, dist)
 self.y_oof.to_csv('{0}/{1}.y_oof.csv'.format(dist, self.name), index=False)
 self.score.to_csv('{0}/{1}.score.csv'.format(dist, self.name), header=True)
 def load(self, src: str = '.'):
self.models = []
for i_fold in range(self.n_splits):
self.models.append(self.load_fold(i_fold, src))
self.y_oof = pd.read_csv('{0}/{1}.y_oof.csv'.format(src, self.name))
self.score = pd.read_csv('{0}/{1}.score.csv'.format(src, self.name), index_col=0 ).iloc[:, 0]
return self
test = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
<drop_column>
class DenseECV(BaseEnsembleCV):
def __init__(self, name: str = 'dense_ecv'):
super().__init__(name=name)
 def fit_fold(self, x_train, y_train, x_val, y_val, i_fold: int, **in_params):
default_parmas = dict(
 lr=0.0001,
epochs=50,
batch_size=32, 
)
 params = {**default_parmas, **in_params}
K.clear_session()
model = Sequential([
Dense(512, input_shape=(x_train.shape[1],)) ,
Activation('relu'),
Dropout(0.2),
Dense(y_train.shape[1]),
Activation('sigmoid'),
])
 es = EarlyStopping(patience=5)
 model.compile(
 optimizer=Adam(lr=params['lr']),
loss='binary_crossentropy', 
)
 with tqdm(desc='k-fold {0}/{1}'.format(i_fold+1, self.n_splits), total=params['epochs'])as pbar:
model.fit(
 x_train, y_train,
epochs=params['epochs'],
batch_size=params['batch_size'],
validation_data=(x_val, y_val),
callbacks=[es, ProgressBar(pbar)],
verbose=0, 
)
 return model
def save_fold(self, model, i_fold, dist):
model.save('{0}/{1}.model_{2}.h5'.format(dist, self.name, i_fold))
def load_fold(self, i_fold, src):
return load_model('{0}/{1}.model_{2}.h5'.format(src, self.name, i_fold))
ctlVehicle_idx = train["cp_type"] != "ctl_vehicle"
train = train.loc[ctlVehicle_idx].reset_index(drop=True)
 train = train.drop("cp_type", axis=1)
 train_target = train_target.loc[ctlVehicle_idx].reset_index(drop=True )<define_variables>
class ElasticNetECV(BaseEnsembleCV):
def __init__(self, verbose: bool = True, name: str = 'elasticnet_ecv'):
super().__init__(verbose=verbose, name=name)
 def fit_fold(self, x_train, y_train, *_, **in_params):
default_params = dict(
 alpha=0.001,
l1_ratio=0.5,
selection='random', 
)
 params = {**default_params, **in_params}
model = MultiTaskElasticNet(random_state=31, **params)
 return model.fit(x_train, y_train)
 def save_fold(self, model, i_fold, dist):
joblib.dump(model, '{0}/{1}.model_{2}.joblib'.format(dist, self.name, i_fold))
def load_fold(self, i_fold, src):
return joblib.load('{0}/{1}.model_{2}.joblib'.format(src, self.name, i_fold))
gcols = [g for g in train.columns if "g-" in g]
ccols = [c for c in train.columns if "c-" in c]
cpcols = [cp for cp in train.columns if "cp_" in cp]<categorify>
class BertFineTuningECV(BaseEnsembleCV):
def __init__(self, pretrained_path: str, seq_len: int = 512, name: str = 'bert_ecv'):
super().__init__(name=name)
 self.pretrained_path = pretrained_path
self.seq_len = seq_len
def build_model(self, params)-> Model:
s = self.seq_len
with tf.device('/cpu:0'):
x = Input(self.seq_len * 3, dtype=tf.int32, name='x')
 input_word_ids = Lambda(lambda x: x[:, :s], output_shape=(s,))(x)
 attention_mask = Lambda(lambda x: x[:, s:s*2], output_shape=(s,))(x)
 token_type_ids = Lambda(lambda x: x[:, s*2:], output_shape=(s,))(x)
 bert_layer = TFBertModel.from_pretrained(self.pretrained_path)
 last_hidden_layer, _ = bert_layer([input_word_ids, attention_mask, token_type_ids])
 pooled = GlobalAveragePooling1D()(last_hidden_layer)
 pooled = Dropout(0.2 )(pooled)
 y = Dense(30, activation='sigmoid', name='output' )(pooled)
 model = Model(inputs=x, outputs=y)
 model.compile(
 optimizer=Adam(lr=params['lr']),
loss='binary_crossentropy', 
)
 return model
def fit_fold(self, x_train, y_train, x_val, y_val, i_fold, **in_params):
default_params = dict(
 lr=0.00003,
epochs=5,
batch_size=8, 
)
 params = {**default_params, **in_params}
es = EarlyStopping(patience=5)
 cp = ModelCheckpoint('{0}.model_{1}.h5'.format(self.name, i_fold), save_best_only=True, save_weights_only=True)
 K.clear_session()
model = self.build_model(params)
 h = model.fit(
 x_train, y_train,
epochs=params['epochs'],
batch_size=params['batch_size'],
validation_data=(x_val, y_val),
callbacks=[es, cp],
verbose=1, 
)
 pd.DataFrame(h.history ).to_csv('{0}_history_{1}.csv'.format(self.name, i_fold), index=False)
 return model
def save_fold(self, model, i_fold, dist):
return
def load_fold(self, i_fold, src):
K.clear_session()
model = self.build_model(dict(lr=0.00003))
model.load_weights('{0}/{1}.model_{2}.h5'.format(src, self.name, i_fold))
return model
ctlVehicle_test = test["cp_type"] == "ctl_vehicle"
test = test.drop("cp_type", axis=1)
 train["cp_time"]=train["cp_time"].map({24: 1, 48: 2, 72: 3})
 train["cp_dose"] = train["cp_dose"].map({"D1": 1, "D2": 2})
 test["cp_time"]= test["cp_time"].map({24: 1, 48: 2, 72: 3})
 test["cp_dose"]=test["cp_dose"].map({"D1": 1, "D2": 2})
 train<feature_engineering>
use_model = tensorflow_hub.load('.. /input/universalsentenceencoderlarge4')
 bert_tokenizer = BertTokenizer.from_pretrained('.. /input/transformers/bert-base-uncased' )
for col in train.iloc[:, 3:].columns:
percent = train[col].quantile([0.01, 0.99] ).values
train[col] = np.clip(train[col], percent[0], percent[1])
 for col in test.iloc[:, 3:].columns:
percent = test[col].quantile([0.01, 0.99] ).values
test[col] = np.clip(test[col], percent[0], percent[1] )<train_model>
use_preprocess = Pipeline(steps=[ 
('USE_question_title', UniversalSentenceEncoderEncoder('question_title', use_model)) , 
('USE_question_body', UniversalSentenceEncoderEncoder('question_body', use_model)) , 
('USE_answer', UniversalSentenceEncoderEncoder('answer', use_model)) , 
('distance_use_question_title-question_body', DistanceEngineerer('question_title_use', 'question_body_use')) , 
('distance_use_question_title-answer', DistanceEngineerer('question_title_use', 'answer_use')) , 
('distance_use_question_body-answer', DistanceEngineerer('question_body_use', 'answer_use')) , 
('onehost_encode_and_drop_columns', Pipeline(steps=[ 
('extrace_sld', ColumnTransformer({
'host': SecondLevelDomainExtracter() ,
})) , 
('onehot_encode_host', WrappedOneHotEncoder('host')) , 
('onehot_encode_category', WrappedOneHotEncoder('category')) ,
] ).fit(features_train)) , 
('drop_columns', ColumnDropper([
'qa_id', 'category', 'host', 'question_title', 'question_body', 'question_user_name', 'question_user_page',
'answer', 'answer_user_name', 'answer_user_page', 'url',
])) ,
])
 use_densenn = Pipeline(steps=[ 
('preprocess', use_preprocess), 
('estimate', DenseECV()),
])
 use_elasticnet = Pipeline(steps=[ 
('preprocess', use_preprocess), 
('estimate', ElasticNetECV())
 ])
 bert_finetuning = Pipeline(steps=[ 
('tokenize', QATokenizer(bert_tokenizer)) , 
('estimate', BertFineTuningECV('.. /input/transformers/bert-base-uncased')) ,
] )
g_pca = PCA(n_components=0.99)
 c_pca = PCA(n_components=0.99)
 train_test_g_concat = pd.concat([train[gcols], test[gcols]], axis=0)
 train_test_c_concat = pd.concat([train[ccols], test[ccols]], axis=0)
 g_pca.fit(train_test_g_concat)
 c_pca.fit(train_test_c_concat)
 train_gtrans = pd.DataFrame(g_pca.transform(train[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=train.index)
 test_gtrans = pd.DataFrame(g_pca.transform(test[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=test.index)
 train_ctrans = pd.DataFrame(c_pca.transform(train[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=train.index)
 test_ctrans = pd.DataFrame(c_pca.transform(test[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=test.index)
 g_pca.n_components_, c_pca.n_components_<concatenate>
question_title = features_train['question_title']
train = pd.concat([train_gtrans, train_ctrans, train[cpcols]], axis=1)
 test = pd.concat([test_gtrans, test_ctrans, test[cpcols]], axis=1)
 train<drop_column>
_ = bert_finetuning['estimate'].load('.. /input/google-quest-challenge-trained-models' )
train_target = train_target.drop("sig_id", axis=1)
 train_target<choose_model_class>
_ = use_densenn['estimate'].load('.. /input/google-quest-challenge-trained-models' )
def create_shallow_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(128, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=adam)
 return model<choose_model_class>
_ = use_elasticnet['estimate'].load('.. /input/google-quest-challenge-trained-models' )
def create_mid_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_adamw)
 return model<choose_model_class>
average_spearmanr(
 pd_average([
bert_finetuning['estimate'].y_oof,
use_densenn['estimate'].y_oof,
use_elasticnet['estimate'].y_oof,
] ).values,
targets_train.values
)
def create_deep_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="selu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="elu")) ,
L.BatchNormalization() ,
L.Dropout(0.1),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_radam)
 return model<train_on_grid>
average_spearmanr(
 pd_rank_average([
bert_finetuning['estimate'].y_oof,
use_densenn['estimate'].y_oof,
use_elasticnet['estimate'].y_oof,
] ).values,
targets_train.values
)
predictions = []
kf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 tf.random.set_seed(42)
 for fold_id,(train_idx, valid_idx)in enumerate(kf.split(train, train_target)) :
model1 = create_shallow_model()
model2 = create_mid_model()
model3 = create_deep_model()
history1 = model1.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model1_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 1, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history1.history["loss"]), min(history1.history["val_loss"])))
 history2 = model2.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model2_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 2, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history2.history["loss"]), min(history2.history["val_loss"])))
 history3 = model3.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model3_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 3, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history3.history["loss"]), min(history3.history["val_loss"])))
 model1.load_weights("model1_fold" + str(fold_id)+ ".h5")
 model2.load_weights("model2_fold" + str(fold_id)+ ".h5")
 model3.load_weights("model3_fold" + str(fold_id)+ ".h5")
 pred1 = model1.predict(test)
 pred2 = model2.predict(test)
 pred3 = model3.predict(test)
 predictions.append(np.average([pred1, pred2, pred3], weights=[0.15, 0.7, 0.15], axis=0))<prepare_output>
def to_submission(y: pd.DataFrame)-> pd.DataFrame:
return pd.concat([features_test.loc[:, 'qa_id'], y], axis=1 )
<load_from_csv><EOS> 
WrappedMinMaxScaler().fit_transform(pd_average([
bert_finetuning.predict(features_test),
use_densenn.predict(features_test),
use_elasticnet.predict(features_test),
])).pipe(to_submission ).to_csv('submission.csv', index=False )