Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<count_values>
LABEL_SMOOTHING=0.001
SEEDS = [120,2524]
SPLITS = 7
BATCH_SIZE = 64
EPOCHS = 60
VARIANCE_THRESHOLD = True
ADD_NON_TRAIN = True
ADD_SEEDS = SEEDS
ADD_SPLITS = SPLITS
ADD_BATCH_SIZE = BATCH_SIZE
ADD_EPOCHS = EPOCHS
RUN_SNN = True
SNN_SEEDS = SEEDS + [42]
SNN_SPLITS = SPLITS
SNN_BATCH_SIZE = BATCH_SIZE
SNN_EPOCHS = EPOCHS
RUN_NN = True
NN_SEEDS = SEEDS
NN_SPLITS = SPLITS
NN_BATCH_SIZE = BATCH_SIZE
NN_EPOCHS = EPOCHS
RUN_NN2 = True
NN2_SEEDS = SEEDS
NN2_SPLITS = SPLITS
NN2_BATCH_SIZE = BATCH_SIZE
NN2_EPOCHS = EPOCHS
RUN_RNN = True
RNN_SEEDS = SEEDS
RNN_SPLITS = SPLITS
RNN_BATCH_SIZE = 128
RNN_EPOCHS = EPOCHS
RUN_TABNET = True
TAB_SEEDS = SEEDS
TAB_SPLITS = SPLITS
TAB_BATCH_SIZE = BATCH_SIZE
TAB_EPOCHS = EPOCHS
RUN_STACKING = True
STK_SEEDS = SEEDS + [42]
STK_SPLITS = SPLITS
STK_EPOCHS = EPOCHS
STK_BATCH_SIZE = BATCH_SIZE
RE_RUN_NN = True
RE_NN_SEEDS = SEEDS
print("True Distribution:")
 print(pd.value_counts(X_train['AdoptionSpeed'], normalize=True ).sort_index())
 print("
Train Predicted Distribution:")
 print(pd.value_counts(train_predictions_lgb, normalize=True ).sort_index())
 print("
Test Predicted Distribution:")
 print(pd.value_counts(test_predictions_lgb, normalize=True ).sort_index() )<concatenate>
import pandas as pd
import numpy as np
import random
import os
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
from tensorflow_addons.layers import WeightNormalization
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
df = pd.concat([X_train, X_test], axis=0)
 df.head(2 )<concatenate>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 torch.manual_seed(seed)
 session_conf = tf.compat.v1.ConfigProto(
 intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess)
 def swish(x):
return x * K.sigmoid(x)
 def show_metrics(valid_preds, show_each=True):
metrics = []
for _target in valid_preds.columns:
logloss = log_loss(train_targets_scored.iloc[:,1:].loc[:, _target], valid_preds.loc[:, _target])
 metrics.append(logloss)
 if show_each:
print(f'column: {_target}, log loss: {logloss}')
 print(f'OOF Metric: {np.mean(metrics)}')
 return metrics
df_ = pd.concat([train,test],axis=0 )<categorify>
train_features = pd.read_csv(f'.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv(f'.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(f'.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv(f'.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv(f'.. /input/lish-moa/sample_submission.csv' )
word_vec_size = 300
max_words = 100
max_word_features = 25000
def transform_text(text, tokenizer):
tokenizer.fit_on_texts(text)
 text_emb = tokenizer.texts_to_sequences(text)
 text_emb = sequence.pad_sequences(text_emb, maxlen=max_words)
 return text_emb
desc_tokenizer = text.Tokenizer(num_words=max_word_features)
 desc_embs = transform_text(df_["Description"].astype(str), desc_tokenizer )<define_variables>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
text_mode = "fasttext"
if text_mode == "fasttext":
embedding_file = ".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec"
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(embedding_file))
word_index = desc_tokenizer.word_index
print('Word index len:', len(word_index))
vocabulary_size = min(max_word_features, len(word_index)) + 1
text_embs = np.zeros(( vocabulary_size, word_vec_size))
for word, i in word_index.items() :
if i >= max_word_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: text_embs[i] = embedding_vector
del(embeddings_index)
 elif text_mode == "word2vec":
embedding_file = ".. /input/word2vec-google/GoogleNews-vectors-negative300.bin"
print("Loading word vectors...")
 word_vectors = KeyedVectors.load_word2vec_format(embedding_file, binary=True)
 print("Matching word vectors...")
 EMBEDDING_DIM=300
word_index = desc_tokenizer.word_index
vocabulary_size=min(len(word_index)+1,max_word_features)
 text_embs = np.zeros(( vocabulary_size, EMBEDDING_DIM))
for word, i in word_index.items() :
if i>=max_word_features:
continue
try:
embedding_vector = word_vectors[word]
text_embs[i] = embedding_vector
except KeyError:
text_embs[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)
 del(word_vectors )<count_missing_values>
_X_train = train_features.copy()
_X_test = test_features.copy()
_y_train = train_targets_scored.copy()
np.sum(pd.isnull(df))<define_variables>
def deal_with_cp_type(_X_train, _X_test):
train_trt_index = _X_train.cp_type == 'trt_cp'
test_trt_index = _X_test.cp_type == 'trt_cp'
_X_train = _X_train[train_trt_index]
_X_test = _X_test
del _X_train['cp_type']
del _X_test['cp_type']
return _X_train, _X_test, train_trt_index, test_trt_index
_X_train, _X_test, train_trt_index, test_trt_index = deal_with_cp_type(_X_train, _X_test)
 def deal_with_cp_dose(_X_train, _X_test):
_X_train = pd.concat([_X_train, pd.get_dummies(_X_train.cp_dose)], axis=1)
 del _X_train['cp_dose']
_X_test = pd.concat([_X_test, pd.get_dummies(_X_test.cp_dose)], axis=1)
 del _X_test['cp_dose']
return _X_train, _X_test
_X_train, _X_test = deal_with_cp_dose(_X_train, _X_test)
 def deal_with_sig_id(_X_train, _X_test):
del _X_train['sig_id']
del _X_test['sig_id']
return _X_train, _X_test
_X_train, _X_test = deal_with_sig_id(_X_train, _X_test)
 def deal_with_y(_y_train, train_trt_index):
_y_train = _y_train[train_trt_index]
del _y_train['sig_id']
return _y_train
_y_train = deal_with_y(_y_train, train_trt_index)
 BASE_COLS = ['cp_time', 'D1', 'D2']
_X_train_dae = _X_train.copy()
_X_test_dae = _X_test.copy()
cat_vars = ["Type", "Breed1", "Breed2", "Color1", "Color2", "Color3", "Gender", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized", "Health", "State"]
cont_vars = ["Fee", "PhotoAmt", "VideoAmt", "Age", "Quantity",'RescuerID_CNT']<categorify>
_X_train = _X_train.reset_index()
def preproc(df):
global cont_vars
for var in cat_vars:
df[var] = LabelEncoder().fit_transform(df[var])
 for var in cont_vars:
df[var] = MinMaxScaler().fit_transform(df[var].values.reshape(-1,1))
return df<normalization>
non_scored_ones = pd.DataFrame()
non_scored_ones['col_name'] = ''
non_scored_ones['item_counts'] = 0
for col in train_targets_nonscored.columns[1:]:
item_counts = len(train_targets_nonscored[train_targets_nonscored[col] == 1])
 non_scored_ones = non_scored_ones.append({'col_name':col, 'item_counts':item_counts}, ignore_index=True)
 non_scored_target_cols = non_scored_ones[non_scored_ones.item_counts > 10].col_name
y_non_train = train_targets_nonscored[non_scored_target_cols]
y_non_train = y_non_train[train_trt_index].reset_index(drop=True )
df_scaled = preproc(df)
 train_df = df_scaled[:len(train)]
test_df = df_scaled[len(train):]
len(train_df), len(test_df )<feature_engineering>
def yield_pca(_X_train, _X_test, prefix, decomp_cols, comp=50, random_state=42):
pca = PCA(n_components=comp, random_state=random_state)
 pca.fit(pd.concat([_X_train.loc[:,decomp_cols], _X_test.loc[:,decomp_cols]]))
_X_train_PCA = pca.transform(_X_train[decomp_cols])
 _X_test_PCA = pca.transform(_X_test[decomp_cols])
 _X_train_PCA = pd.DataFrame(_X_train_PCA, columns=[f'pca_{prefix}-{i}' for i in range(comp)])
 _X_test_PCA = pd.DataFrame(_X_test_PCA, columns=[f'pca_{prefix}-{i}' for i in range(comp)])
 return _X_train_PCA, _X_test_PCA
_X_train_G_PCA, _X_test_G_PCA = yield_pca(_X_train, _X_test, 'G', GENES, 90)
 _X_train_C_PCA, _X_test_C_PCA = yield_pca(_X_train, _X_test, 'C', CELLS, 27)
 _X_train_G_PCA_Dense, _X_test_G_PCA_Dense = yield_pca(_X_train, _X_test, 'Gd', GENES, 8)
 _X_train_C_PCA_Dense, _X_test_C_PCA_Dense = yield_pca(_X_train, _X_test, 'Cd', CELLS, 1)
 _X_train = pd.concat([_X_train, _X_train_G_PCA], axis=1)
 _X_train = pd.concat([_X_train, _X_train_C_PCA], axis=1)
 _X_train = pd.concat([_X_train, _X_train_G_PCA_Dense], axis=1)
 _X_train = pd.concat([_X_train, _X_train_C_PCA_Dense], axis=1)
 _X_test = pd.concat([_X_test, _X_test_G_PCA], axis=1)
 _X_test = pd.concat([_X_test, _X_test_C_PCA], axis=1)
 _X_test = pd.concat([_X_test, _X_test_G_PCA_Dense], axis=1)
 _X_test = pd.concat([_X_test, _X_test_C_PCA_Dense], axis=1)
 PCA_G = [col for col in _X_train.columns if col.startswith('pca_G-')] + [col for col in _X_train.columns if col.startswith('pca_Gd-')]
PCA_C = [col for col in _X_train.columns if col.startswith('pca_C-')] + [col for col in _X_train.columns if col.startswith('pca_Cd-')]
def get_keras_data(df, description_embeds):
X = {var: df[var].values for var in cont_vars+cat_vars}
X["description"] = description_embeds
for i in range(256): X[f"img_{i}"] = df[f"img_{i}"]
return X<import_modules>
def make_feature(data, target_feats, create, prefix):
target_data = data.loc[:,target_feats]
invoker = getattr(target_data, create)
 return pd.DataFrame(invoker(axis=1), columns=[f'{prefix}_{create}'])
 for method in ['sum', 'mean', 'std', 'skew', 'kurt', 'median']:
_X_train = pd.concat([_X_train, make_feature(_X_train, GENES+CELLS, method, 'gce')], axis = 1)
 _X_train = pd.concat([_X_train, make_feature(_X_train, GENES, method, 'ge')], axis = 1)
 _X_test = pd.concat([_X_test, make_feature(_X_test, GENES+CELLS, method, 'gce')], axis = 1)
 _X_test = pd.concat([_X_test, make_feature(_X_test, GENES, method, 'ge')], axis = 1)
 GC_EX = [col for col in _X_train.columns if col.startswith('gce_')]
G_EX = [col for col in _X_train.columns if col.startswith('ge_')]
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class QWKEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.history = []
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, batch_size=1000, verbose=0)
 y_pred = eval_predict(self.y_val, y_pred)
 score = quadratic_weighted_kappa(self.y_val, y_pred)
 print("QWK - epoch: %d - score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history): self.model.save('checkpoint.h5')
 def eval_predict(y=[], y_pred=[], coeffs=None, ret_coeffs=False):
optR = OptimizedRounder()
if not coeffs:
optR.fit(y_pred.reshape(-1,), y)
 coeffs = optR.coefficients()
if ret_coeffs: return optR.coefficients()
return optR.predict(y_pred, coeffs ).reshape(-1, )<compute_test_metric>
train_col_names = _X_train.columns
test_col_names = _X_test.columns
TRANSFORM_TARGET_COLS = BASE_COLS + GENES + CELLS + PCA_G + PCA_C + G_EX + GC_EX
def rank_gauss(_X_train, _X_test, cols=TRANSFORM_TARGET_COLS, random_state=42):
qt = QuantileTransformer(random_state=random_state, output_distribution='normal')
 qt.fit(pd.concat([_X_train[cols], _X_test[cols]]))
_X_train[cols] = qt.transform(_X_train[cols])
 _X_test[cols] = qt.transform(_X_test[cols])
 return _X_train, _X_test
def standard_scaler(_X_train, _X_test, cols=TRANSFORM_TARGET_COLS):
ss = StandardScaler()
ss.fit(pd.concat([_X_train[cols], _X_test[cols]]))
_X_train[cols] = ss.transform(_X_train[cols])
 _X_test[cols] = ss.transform(_X_test[cols])
 return _X_train, _X_test
_X_train, _X_test = rank_gauss(_X_train, _X_test)
 _X_train, _X_test = standard_scaler(_X_train, _X_test)
 _X_train.columns = train_col_names
_X_test.columns = test_col_names
def rmse(y, y_pred):
return K.sqrt(K.mean(K.square(y-y_pred), axis=-1))
def get_model(emb_n=10, dout=.25, batch_size=1000):
inps = []
embs = []
nums = []
for var in cat_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 embs.append(( Embedding(df[var].max() +1, emb_n )(inp)))
 for var in cont_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 nums.append(( inp))
desc_inp = Input(shape=(max_words,), name="description")
 inps.append(desc_inp)
 emb_desc = Embedding(vocabulary_size, word_vec_size, weights=[text_embs] )(desc_inp)
 emb_desc = SpatialDropout1D (.4 )(emb_desc)
 emb_desc = Reshape(( max_words, word_vec_size, 1))(emb_desc)
 filter_sizes=[1,3,1]
convs = []
for filter_size in filter_sizes:
conv = Conv2D(32, kernel_size=(filter_size, word_vec_size),
kernel_initializer="normal", activation="relu" )(emb_desc)
 convs.append(MaxPool2D(pool_size=(max_words-filter_size+1, 1))(conv))
img_fts = []
for i in range(256):
inp = Input(shape=[1], name=f"img_{i}")
 inps.append(inp)
 img_fts.append(( inp))
img_fts = concatenate(img_fts)
 img_fts = Dropout (.5 )(img_fts)
 img_fts = BatchNormalization()(img_fts)
 img_fts = Dense(64, activation="relu" )(img_fts)
 convs = concatenate(convs)
 convs = Flatten()(convs)
 convs = BatchNormalization()(convs)
 embs = Flatten()(concatenate(embs))
embs = Dropout(dout )(Dense(64, activation="relu", kernel_initializer="he_normal" )(embs))
nums = concatenate(nums)
 nums = Dense(32, activation="relu" )(nums)
 nums = Dropout(0.5 )(nums)
 x = concatenate([embs, nums, convs, img_fts])
 x = BatchNormalization()(x)
 dense_n = [256, 64]
for n in dense_n:
x = BatchNormalization()(x)
 x = Dense(n, activation="relu", kernel_initializer="he_normal" )(x)
 x = BatchNormalization()(x)
 x = Dropout (.2 )(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inps, outputs=out)
 opt = Adam()
model.compile(optimizer=opt, loss='mse',)
 return model 
<prepare_x_and_y>
if VARIANCE_THRESHOLD:
vt_cols = vt_cols.values.tolist() + BASE_COLS + PCA_G + PCA_C + GC_EX + G_EX
_X_train_nl = pd.concat([_X_train.iloc[:,:1], _X_train.loc[:,vt_cols]], axis=1)
 _X_test_nl = _X_test.loc[:,vt_cols]
vt_cols_leaky = vt_cols_leaky.values.tolist() + BASE_COLS + PCA_G + PCA_C + GC_EX + G_EX
_X_train = pd.concat([_X_train.iloc[:,:1], _X_train.loc[:,vt_cols_leaky]], axis=1)
 _X_test = _X_test.loc[:,vt_cols_leaky]
GENES = [col for col in _X_train.columns if col.startswith('g-')]
CELLS = [col for col in _X_train.columns if col.startswith('c-')]
nfolds=5
folds = StratifiedKFold(n_splits=nfolds,shuffle=True, random_state=15)
 avg_train_kappa = 0
avg_valid_kappa = 0
batch_size=1000
coeffs=None
x_test = get_keras_data(test_df, desc_embs[len(train_df):])
 adoptions_keras = np.zeros(( len(test_df),))
oof_train_keras = np.zeros(( train_df.shape[0]))
i =0
for train_idx, valid_idx in folds.split(train_df[cat_vars+cont_vars], train_df["AdoptionSpeed"]):
x_train = get_keras_data(train_df.iloc[train_idx], desc_embs[train_idx])
 x_valid = get_keras_data(train_df.iloc[valid_idx], desc_embs[valid_idx])
 y_train, y_valid = train_df["AdoptionSpeed"][train_idx].values, train_df["AdoptionSpeed"][valid_idx].values
model = get_model()
clr_tri = CyclicLR(base_lr=2e-3, max_lr=4e-2, step_size=len(train_df)//batch_size, mode="triangular2")
 qwk_eval = QWKEvaluation(validation_data=(x_valid, y_valid), interval=1)
 history = model.fit(x_train, y_train, batch_size=batch_size, validation_data=(x_valid, y_valid),
epochs=20, callbacks=[clr_tri, qwk_eval])
 model.load_weights('checkpoint.h5')
 y_train_pred = eval_predict(y_train, model.predict(x_train, batch_size=1000))
y_valid_pred = eval_predict(y_valid, model.predict(x_valid, batch_size=1000))
avg_train_kappa += quadratic_weighted_kappa(y_train_pred, y_train)
 avg_valid_kappa += quadratic_weighted_kappa(y_valid_pred, y_valid)
 coeffs = eval_predict(y_valid, model.predict(x_valid, batch_size=1000), ret_coeffs=True)
 adoptions_keras += model.predict(x_test, batch_size=batch_size ).reshape(-1,)
 oof_train_keras[valid_idx] = y_valid_pred
print("
avg train kappa:", avg_train_kappa/nfolds,)
 print("
avg valid kappa:", avg_valid_kappa/nfolds, )<count_values>
_X_train = _X_train.set_index('index')
 X_train = _X_train.copy()
X_test = _X_test.copy()
_X_train_nl = _X_train_nl.set_index('index')
 X_train_nl = _X_train_nl.copy()
X_test_nl = _X_test_nl.copy()
y_train = _y_train.copy()
Counter(X_train['AdoptionSpeed'] )<count_values>
def train_and_predict(name, model_func, _X_i_train, _y_i_train, _X_i_test, orig_targets, result_template,
seeds, splits, epochs, batch_size, shuffle_rows=False, pick_col_size=800,
do_show_metrics=True, show_each_metrics=True):
st = time.time()
is_list = isinstance(_X_i_train, list)
 val_result = orig_targets.copy()
val_result.loc[:, :] = 0
sub_result = result_template.copy()
sub_result.loc[:, 1:] = 0
for h, seed in enumerate(seeds):
seed_everything(seed)
 for i,(train_idx, valid_idx)in enumerate(MultilabelStratifiedKFold(n_splits=splits, random_state=seed, shuffle=True)
.split(_y_i_train, _y_i_train)) :
print(f'Fold {i+1}')
 if is_list:
_X_train = [_X_i_train[0].loc[:,:].values[train_idx], _X_i_train[1].loc[:,:].values[train_idx]]
_X_valid = [_X_i_train[0].loc[:,:].values[valid_idx], _X_i_train[1].loc[:,:].values[valid_idx]]
else:
_X_train = _X_i_train.loc[:,:].values[train_idx]
_X_valid = _X_i_train.loc[:,:].values[valid_idx]
_y_train = _y_i_train.values[train_idx]
_y_valid = _y_i_train.values[valid_idx]
if is_list:
model = model_func(len(_X_i_train[0].columns), len(_X_i_train[1].columns))
else:
model = model_func(len(_X_i_train.columns))
model.fit(_X_train, _y_train,
validation_data=(_X_valid, _y_valid),
epochs=epochs, batch_size=batch_size,
callbacks=[
ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=3, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'{name}_{seed}_{i}.hdf5', monitor = 'val_logloss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 , EarlyStopping(monitor = 'val_logloss', min_delta = 1e-4, patience = 5, mode = 'min', baseline = None, restore_best_weights = True)
 ], verbose=2)
 model.load_weights(f'{name}_{seed}_{i}.hdf5')
 val_result.iloc[_y_i_train.iloc[valid_idx,:].index.values, :] += model.predict(_X_valid)
 if is_list:
sub_result.loc[test_trt_index, sub_result.columns[1:]] += model.predict([_X_i_test[0].loc[test_trt_index, :], _X_i_test[1].loc[test_trt_index, :]])
 else:
sub_result.loc[test_trt_index, sub_result.columns[1:]] += model.predict(_X_i_test.loc[test_trt_index, :])
 print('')
 tmp_result = val_result.copy()
tmp_result.iloc[:,1:] = val_result.iloc[:,1:] /(h + 1)
 print(f' ---- seed:{seed}, ensemble:{h + 1}')
 if do_show_metrics:
_ = show_metrics(tmp_result, show_each=False)
 val_result.iloc[:,1:] = val_result.iloc[:,1:] / len(seeds)
 if do_show_metrics:
metrics = show_metrics(val_result, show_each=show_each_metrics)
 else:
metrics = None
sub_result.iloc[:, 1:] = sub_result.iloc[:, 1:] /(len(seeds)* splits)
 print(f' elapsed: {time.time() - st}')
 return sub_result, val_result, metrics
Counter(X_train['AdoptionSpeed'] )<predict_on_test>
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -K.mean(y_true*K.log(y_pred)+(1-y_true)*K.log(1-y_pred))
coeffs[0] = 1.645
coeffs[1] = 2.115
coeffs[3] = 2.84
print('True Counter',Counter(X_train['AdoptionSpeed']))
train_pred_keras = eval_predict(y_pred=oof_train_keras, coeffs=list(coeffs)).astype(int)
 print('Train pred Counter',Counter(train_pred_keras))
test_pred_keras = eval_predict(y_pred=adoptions_keras/nfolds, coeffs=list(coeffs)).astype(int)
 print('Test pred Counter',Counter(test_pred_keras))<concatenate>
output_bias=tf.keras.initializers.Constant(-np.log(y_train.mean(axis=0 ).to_numpy()))
train_stack = np.vstack([train_pred_keras, train_predictions_lgb] ).transpose()
test_stack = np.vstack([test_pred_keras, test_predictions_lgb] ).transpose()<import_modules>
def create_add_model(input_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(input_dim, activation='elu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.4))
model.add(WeightNormalization(L.Dense(512, activation='selu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(y_non_train.shape[1], activation='sigmoid',
bias_initializer=tf.keras.initializers.Constant(-np.log(y_non_train.mean(axis=0 ).to_numpy())
))))
model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=1e-3), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model
from sklearn.linear_model import Ridge<prepare_x_and_y>
non_scored_template = sample_submission.copy()
non_scored_template = pd.DataFrame(non_scored_template.pop('sig_id'))
for col in non_scored_target_cols:
non_scored_template[col] = 0 .
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(train_stack.shape[0])
 predictions = np.zeros(test_stack.shape[0])
 qwk_scores = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_stack, X_train['AdoptionSpeed'].values)) :
trn_data, trn_y = train_stack[trn_idx], X_train['AdoptionSpeed'].iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], X_train['AdoptionSpeed'].iloc[val_idx].values
clf = Ridge(alpha=1)
 clf.fit(trn_data, trn_y)
 val_pred = clf.predict(val_data)
 pred_val_y_k = eval_predict(y_pred=val_pred, coeffs=list(coeffs)).astype(int)
 print("Valid Counts = ", Counter(val_y))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(val_y, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 oof[val_idx] = clf.predict(val_data)
 predictions += clf.predict(test_stack)/ folds.n_splits
print('{} cv mean QWK score : {}'.format('Stacked_model_ridge', np.mean(qwk_scores)) )<predict_on_test>
if ADD_NON_TRAIN:
add_result, add_valid_preds, add_metrics = train_and_predict('ADD', create_add_model, X_train, y_non_train, X_test,
train_targets_nonscored[non_scored_target_cols].loc[train_trt_index,:], non_scored_template,
ADD_SEEDS, ADD_SPLITS, ADD_EPOCHS, ADD_BATCH_SIZE, shuffle_rows=False, do_show_metrics=False, show_each_metrics=False )
pred_final = eval_predict(y_pred=predictions, coeffs=list(coeffs)).astype(int )<count_values>
if ADD_NON_TRAIN:
X_non_train = add_valid_preds.copy()
X_non_test = add_result.iloc[:,1:].copy()
cols = X_non_train.columns
X_non_train, X_non_test = rank_gauss(X_non_train, X_non_test, cols)
 X_non_train, X_non_test = standard_scaler(X_non_train, X_non_test, cols )
Counter(pred_final )<count_values>
def create_autoencoder(input_dim):
input_vector = L.Input(shape=(input_dim,))
encoded = L.Dense(1024, activation='elu' )(input_vector)
 encoded = L.Dense(2048, activation='elu' )(encoded)
 decoded = L.Dense(1024, activation='elu' )(encoded)
 decoded = L.Dense(input_dim, activation='elu' )(decoded)
 autoencoder = M.Model(input_vector, decoded)
 autoencoder.compile(optimizer=tf.optimizers.Adam(lr=0.0012, amsgrad=True), loss='mse')
 return autoencoder
Counter(test_pred_keras )<count_values>
mu, sigma = 0, 0.1
seed_everything(128)
 dae_train = pd.concat([_X_train_dae, _X_test_dae])
 noise = np.random.normal(mu, sigma, [dae_train.shape[0], dae_train.shape[1]])
 noised_train = dae_train + noise
autoencoder = create_autoencoder(dae_train.shape[1])
 autoencoder.fit(noised_train, dae_train,
epochs=500,
batch_size=128,
callbacks=[
ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience = 3, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'dae.hdf5', monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 , EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 8, mode = 'min', baseline = None, restore_best_weights = True)],
shuffle=True,
validation_split=0.2)
 autoencoder.load_weights(f'dae.hdf5')
Counter(test_predictions_lgb )<save_to_csv>
encoder = M.Model(autoencoder.input, autoencoder.layers[2].output)
 __X_train_dae = pd.DataFrame(encoder.predict(_X_train_dae))
__X_test_dae = pd.DataFrame(encoder.predict(_X_test_dae))
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': pred_final.astype(np.int32)})
 submission.head()
submission.to_csv('submission.csv', index=False )<install_modules>
vh_dae = VarianceThreshold(0.2)
 vh_dae.fit_transform(pd.concat([__X_train_dae, __X_test_dae]))
dae_cols = __X_train_dae.columns[vh_dae.get_support() ]
print(len(dae_cols))
X_train_dae = __X_train_dae.loc[:, dae_cols]
X_test_dae = __X_test_dae.loc[:, dae_cols]
!pip install optuna==0.18.1<import_modules>
X_train_dae = pd.concat([X_train_dae, X_non_train.reset_index(drop=True)], axis=1)
 X_test_dae = pd.concat([X_test_dae, X_non_test.reset_index(drop=True)], axis=1 )
optuna.__version__<set_options>
_X_train_G_PCA_dae, _X_test_G_PCA_dae = yield_pca(_X_train, _X_test, 'Gd', GENES, 90, 128)
 _X_train_C_PCA_dae, _X_test_C_PCA_dae = yield_pca(_X_train, _X_test, 'Cd', CELLS, 27, 128)
 _X_train_G_PCA_Dense_dae, _X_test_G_PCA_Dense_dae = yield_pca(_X_train, _X_test, 'Gd', GENES, 8, 128)
 _X_train_C_PCA_Dense_dae, _X_test_C_PCA_Dense_dae = yield_pca(_X_train, _X_test, 'Cd', CELLS, 1, 128)
 X_train_dae = pd.concat([X_train_dae, _X_train_G_PCA_dae, _X_train_C_PCA_dae, _X_train_G_PCA_Dense, _X_train_C_PCA_Dense_dae], axis=1)
 X_test_dae = pd.concat([X_test_dae, _X_test_G_PCA_dae, _X_test_C_PCA_dae, _X_test_G_PCA_Dense, _X_test_C_PCA_Dense_dae], axis=1)
 X_train_dae, X_test_dae = rank_gauss(X_train_dae, X_test_dae, X_train_dae.columns, random_state=128)
 X_train_dae, X_test_dae = standard_scaler(X_train_dae, X_test_dae, X_train_dae.columns )
np.random.seed(369 )<compute_test_metric>
def create_simple_nn(input_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(input_dim, activation='selu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(512, activation='swish')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(206, activation='sigmoid', bias_initializer=output_bias)))
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=0.01), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_test_metric>
if RUN_SNN:
snn_result, snn_valid_preds, snn_metrics = train_and_predict('SNN', create_simple_nn, X_train_dae, _y_train, X_test_dae,
train_targets_scored.iloc[:,1:], sample_submission,
SNN_SEEDS, SNN_SPLITS, SNN_EPOCHS, SNN_BATCH_SIZE, shuffle_rows=False )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<load_from_csv>
_X_train = pd.concat([X_train, X_non_train], axis=1)
 _X_test = pd.concat([X_test, X_non_test], axis=1 )
print('Train')
 train = pd.read_csv(".. /input/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/state_labels.csv")
 print(states.shape)
 target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )<feature_engineering>
sep_cols = BASE_COLS + GENES
_X_train_1 = _X_train[sep_cols]
_X_train_2 = _X_train.drop(sep_cols,axis=1)
 _X_test_1 = _X_test[sep_cols]
_X_test_2 = _X_test.drop(sep_cols,axis=1)
 _X_train = [_X_train_1, _X_train_2]
_X_test = [_X_test_1, _X_test_2]
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score 
<normalization>
def create_res_model(n_features, n_features_2):
print(f'the input dim is {n_features}, {n_features_2}')
 input_1 = L.Input(shape =(n_features,), name = 'Input1')
 input_2 = L.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = M.Sequential([
L.BatchNormalization() ,
L.Dropout(0.3),
L.Dense(512, activation='elu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(256, activation='elu')
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = L.Concatenate()([input_2, input_3])
 head_2 = M.Sequential([
L.BatchNormalization() ,
L.Dropout(0.3),
L.Dense(n_features_2, activation='relu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(n_features_2, activation='elu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(256, activation='relu'),
L.BatchNormalization() ,
L.Dropout(0.5),
L.Dense(256, activation='selu')
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = L.Average()([input_3, input_4])
 head_3 = M.Sequential([
L.BatchNormalization() ,
L.Dropout(0.3),
L.Dense(256, activation='swish'),
L.BatchNormalization() ,
L.Dense(256, activation='selu'),
L.BatchNormalization() ,
L.Dense(206, activation='sigmoid')
 ],name='Head3')
 output = head_3(input_4_avg)
 model = M.Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer=tf.optimizers.Adam(lr=0.002),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model
SVD_FEATURES = 200
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 print("X(tfidf):", X.shape)
 svd = TruncatedSVD(n_components=SVD_FEATURES)
 svd.fit(X)
 X = svd.transform(X)
 print("X(svd):", X.shape)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(SVD_FEATURES)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(SVD_FEATURES)])
 test = pd.concat(( test, X_test), axis=1)
 print("train:", train.shape )<prepare_x_and_y>
if RUN_NN:
nn_result, nn_valid_preds, nn_metrics = train_and_predict('RES', create_res_model, _X_train, y_train, _X_test,
train_targets_scored.iloc[:,1:], sample_submission,
NN_SEEDS, NN_SPLITS, NN_EPOCHS, NN_BATCH_SIZE, shuffle_rows=False )
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores<data_type_conversions>
if ADD_NON_TRAIN:
_X_train = []
_X_train.append(X_train.copy())
 _X_train.append(X_non_train.copy())
 _X_test = []
_X_test.append(X_test.copy())
 _X_test.append(X_non_test.copy())
 _X_train[1] = pd.concat([_X_train[1], _X_train[0][CELLS]], axis=1)
 _X_test[1] = pd.concat([_X_test[1], _X_test[0][CELLS]], axis=1)
 _X_train[0] = _X_train[0].drop(CELLS, axis=1)
 _X_test[0] = _X_test[0].drop(CELLS, axis=1 )
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i)for i in range(SVD_FEATURES)]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape )<drop_column>
def create_twohead_model(input_dim1, input_dim2):
print(f'the input dim is {input_dim1}, {input_dim2}')
 input_1 = L.Input(input_dim1)
 x1 = L.BatchNormalization()(input_1)
 x1 = L.Dropout(0.5 )(x1)
 o1 = L.Dense(input_dim1, activation='elu' )(x1)
 input_2 = L.Input(input_dim2)
 x2 = L.BatchNormalization()(input_2)
 x2 = L.Dropout(0.5 )(x2)
 o2 = L.Dense(input_dim2, activation='elu' )(x2)
 x = L.Concatenate()([o1, o2])
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.4 )(x)
 x = L.Dense(512, activation='selu' )(x)
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.5 )(x)
 output = L.Dense(206, activation='sigmoid' )(x)
 model = M.Model([input_1, input_2], output)
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=0.01), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model
train.drop(['VideoAmt', 'Type', 'bounding_importance', 'bounding_confidence'], axis=1, inplace=True)
 test.drop(['VideoAmt', 'Type', 'bounding_importance', 'bounding_confidence'], axis=1, inplace=True )<filter>
if RUN_NN2:
nn2_result, nn2_valid_preds, nn2_metrics = train_and_predict('NN2', create_twohead_model, _X_train, y_train, _X_test,
train_targets_scored.iloc[:,1:], sample_submission,
NN2_SEEDS, NN2_SPLITS, NN2_EPOCHS, NN2_BATCH_SIZE, shuffle_rows=False )
foo = train.dtypes
cat_feature_names = foo[foo == "category"]
cat_features = [train.columns.get_loc(c)for c in train.columns if c in cat_feature_names]<find_best_model_class>
top_feats = [ 0, 1, 2, 4, 6, 9, 11, 12, 13, 15, 16,
17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29,
31, 33, 36, 37, 39, 40, 41, 42, 44, 45, 47,
48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 60,
61, 65, 67, 68, 69, 71, 72, 73, 75, 76, 78,
79, 80, 81, 82, 83, 85, 86, 87, 88, 89, 91,
92, 93, 94, 96, 98, 99, 100, 101, 102, 103, 104,
107, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118,
119, 120, 122, 123, 124, 125, 126, 128, 129, 130, 132,
133, 134, 135, 136, 137, 140, 141, 142, 143, 144, 146,
148, 149, 150, 151, 152, 153, 154, 155, 158, 159, 161,
162, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173,
175, 178, 181, 183, 184, 185, 186, 187, 189, 190, 191,
193, 197, 198, 200, 201, 202, 203, 204, 205, 206, 209,
210, 211, 213, 214, 216, 218, 219, 220, 221, 223, 225,
226, 227, 229, 230, 231, 232, 233, 236, 237, 239, 241,
242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 254,
255, 256, 257, 258, 261, 262, 263, 265, 266, 268, 270,
271, 272, 273, 275, 276, 277, 279, 280, 281, 283, 284,
288, 289, 291, 294, 297, 298, 299, 300, 302, 303, 304,
305, 306, 309, 310, 311, 312, 314, 315, 316, 317, 319,
321, 322, 323, 324, 325, 326, 329, 330, 332, 336, 337,
339, 340, 341, 342, 343, 344, 347, 348, 349, 351, 354,
355, 356, 357, 358, 359, 360, 361, 362, 365, 367, 369,
370, 371, 373, 374, 375, 376, 378, 379, 380, 383, 384,
385, 388, 390, 392, 395, 397, 400, 401, 403, 404, 406,
407, 408, 410, 414, 415, 416, 418, 419, 420, 422, 423,
424, 425, 426, 427, 428, 430, 431, 433, 434, 436, 438,
439, 441, 443, 446, 447, 448, 450, 451, 452, 454, 455,
457, 461, 462, 463, 464, 466, 468, 469, 470, 473, 474,
475, 477, 478, 479, 481, 482, 483, 484, 487, 488, 489,
492, 495, 497, 501, 502, 503, 505, 506, 508, 509, 510,
511, 512, 513, 514, 516, 517, 519, 522, 523, 524, 525,
527, 528, 529, 531, 538, 543, 546, 548, 549, 550, 552,
553, 554, 555, 558, 559, 560, 561, 562, 563, 564, 565,
567, 568, 569, 572, 573, 574, 575, 576, 579, 580, 583,
585, 587, 588, 591, 592, 595, 596, 597, 599, 600, 603,
610, 611, 612, 613, 614, 617, 619, 620, 621, 623, 624,
625, 627, 628, 629, 632, 634, 635, 636, 637, 638, 639,
640, 642, 643, 644, 645, 646, 647, 649, 650, 651, 654,
656, 659, 660, 661, 662, 663, 665, 669, 671, 672, 673,
674, 676, 677, 678, 680, 681, 682, 685, 686, 687, 689,
692, 694, 695, 696, 697, 698, 699, 701, 702, 704, 705,
706, 708, 709, 710, 714, 716, 717, 718, 722, 724, 725,
729, 732, 733, 734, 735, 736, 737, 738, 742, 743, 748,
749, 751, 753, 754, 755, 756, 757, 759, 761, 763, 764,
765, 767, 768, 771, 772, 773, 774, 775, 776, 778, 779,
781, 782, 784, 786, 788, 791, 792, 793, 794, 800, 801,
802, 805, 807, 814, 815, 816, 820, 821, 822, 823, 824,
825, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837,
838, 839, 840, 841, 842, 843, 844, 846, 847, 848, 849,
850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860,
861, 863, 865, 866, 867, 868, 869, 870, 871, 872, 873,
876, 877, 878, 879, 880, 881, 882, 883, 884, 886, 887,
889, 890, 892, 893, 894, 895, 896, 897, 898, 899, 901,
902, 904, 905, 906, 909, 910, 911, 912, 913, 914, 915,
916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926,
927, 928, 929, 930, 931, 932, 933, 934, 938, 941, 942,
944, 948, 949, 950, 951, 954, 955, 956, 957, 958, 959,
960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970,
971, 972, 974, 976, 980, 981, 985, 987, 988, 989, 990,
991, 993, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003,
1004, 1005, 1006, 1010, 1011, 1012, 1013, 1014, 1016, 1017, 1018,
1019, 1020, 1021, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1031,
1032, 1034, 1035, 1037, 1038, 1040, 1041, 1042, 1046, 1047, 1048,
1049, 1050, 1051, 1053, 1055, 1056, 1057, 1058, 1059, 1060, 1062,
1064, 1065, 1066, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1076,
1078, 1079, 1080, 1082, 1083, 1084, 1085, 1086, 1087, 1089, 1092,
1093, 1094, 1095, 1096, 1097, 1099, 1100, 1101, 1102, 1103, 1104,
1105, 1106, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116,
1117, 1118, 1119, 1120, 1121, 1122, 1125, 1126, 1127, 1128, 1131,
1132, 1134, 1136, 1137, 1138, 1140, 1141]
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], 5))
all_coefficients = np.zeros(( 5, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 best_params = {}
tuning_history = []
params = {
'objective': 'regression',
'metric': 'rmse',
}
lgb.train(
 params,
d_train,
num_boost_round=10000,
valid_sets=watchlist,
early_stopping_rounds=100,
verbose_eval=200,
best_params=best_params,
tuning_history=tuning_history)
 pd.DataFrame(tuning_history ).to_csv('./tuning_history.csv')
 best_params['learning_rate'] = 0.05
model = lgb_origin.train(params,
train_set=d_train,
num_boost_round=20000,
valid_sets=watchlist,
early_stopping_rounds=1000,
verbose_eval=1000)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
X_train = pd.concat([X_train_nl, X_non_train], axis=1)
 X_test = pd.concat([X_test_nl, X_non_test], axis=1 )
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<predict_on_test>
X_train_rnn = X_train.iloc[:, top_feats]
X_test_rnn = X_test.iloc[:, top_feats]
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<create_dataframe>
def create_rnn_model(input_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.Reshape(( 1, input_dim)))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.3))
model.add(L.GRU(1024, dropout = 0.5, recurrent_dropout=0.3, return_sequences = True, activation='elu'))
model.add(L.GRU(1024, dropout = 0.5, recurrent_dropout=0.5, activation='selu'))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(206, activation='sigmoid', bias_initializer=output_bias)))
 model.compile(optimizer = tfa.optimizers.Lookahead(tf.optimizers.Adam(0.0015), sync_period = 2),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
if RUN_RNN:
rnn_result, rnn_valid_preds, rnn_metrics = train_and_predict('RNN', create_rnn_model, X_train_rnn, y_train, X_test_rnn,
train_targets_scored.iloc[:,1:], sample_submission,
RNN_SEEDS, RNN_SPLITS, RNN_EPOCHS, RNN_BATCH_SIZE )
quadratic_weighted_kappa(target, train_predictions)
 rmse(target, [r[0] for r in results['train']])
 submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except KeyError:
raise KeyError("Your zip file is missing at least one component")
 self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss = self.log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 y_smo = targets.float() *(1 - LABEL_SMOOTHING)+ 0.5 * LABEL_SMOOTHING
loss = self.loss_fn(output, y_smo)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def log_loss_multi(self, y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = self.log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
def log_loss_score(self, actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss = self.log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 y_smo = targets.float() *(1 - LABEL_SMOOTHING)+ 0.5 * LABEL_SMOOTHING
loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res
submission.to_csv('submission.csv', index=False )<save_to_csv>
def train_and_predict(name, _X_i_train, _y_i_train, _X_i_test, seeds, splits, epochs, batch_size, shuffle_rows=False, pick_col_size=800):
st = time.time()
val_result = train_targets_scored.iloc[:,1:].copy()
val_result.loc[:, :] = 0
sub_result = sample_submission.copy()
sub_result.loc[:, 1:] = 0
all_cols = _X_i_train.columns.values
for h, seed in enumerate(seeds):
seed_everything(seed)
 for i,(train_idx, valid_idx)in enumerate(MultilabelStratifiedKFold(n_splits=splits, random_state=seed, shuffle=True)
.split(_y_i_train, _y_i_train)) :
print(f'Fold {i+1}')
 if shuffle_rows:
seed_everything(seed + i)
 target_cols = np.random.choice(all_cols, size=pick_col_size)
 seed_everything(seed)
 else:
target_cols = all_cols
_X_train = torch.as_tensor(_X_i_train.loc[:,target_cols].values[train_idx])
 _y_train = torch.as_tensor(_y_i_train.values[train_idx])
 _X_valid = torch.as_tensor(_X_i_train.loc[:,target_cols].values[valid_idx])
 _y_valid = torch.as_tensor(_y_i_train.values[valid_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5), mask_type='entmax',
device_name="cuda" if torch.cuda.is_available() else "cpu",
scheduler_params=dict(milestones=[100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 model.fit(X_train=_X_train, y_train=_y_train, X_valid=_X_valid, y_valid=_y_valid, max_epochs=epochs, patience=20, batch_size=1024, virtual_batch_size=batch_size,
num_workers=0, drop_last=False, loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 model.load_best_model()
preds = model.predict(_X_valid)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
val_result.iloc[y_train.iloc[valid_idx,:].index.values, :] += preds
preds = model.predict(torch.as_tensor(_X_i_test.loc[test_trt_index, target_cols].values))
preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
sub_result.loc[test_trt_index, sub_result.columns[1:]] += preds
print('')
 tmp_result = val_result.copy()
tmp_result.iloc[:,1:] = val_result.iloc[:,1:] /(h + 1)
 print(f' ---- seed:{seed}, ensemble:{h + 1}')
 _ = show_metrics(tmp_result, show_each=False)
 val_result.iloc[:,1:] = val_result.iloc[:,1:] / len(seeds)
 metrics = show_metrics(val_result)
 sub_result.iloc[:, 1:] = sub_result.iloc[:, 1:] /(len(seeds)* splits)
 print(f' elapsed: {time.time() - st}')
 return sub_result, val_result, metrics
submission.to_csv('submission.csv', index=False )<set_options>
if RUN_TABNET:
tab_result, tab_valid_preds, tab_metrics = train_and_predict('TAB', X_train, y_train, X_test,
TAB_SEEDS, TAB_SPLITS, TAB_EPOCHS, TAB_BATCH_SIZE )
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'<load_from_csv>
if not RE_RUN_NN:
ens_result.to_csv('submission.csv', index=False )
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<import_modules>
def re_train_and_predict(name, model_func, _X_i_train, _y_i_train, _X_i_test, seeds, splits, epochs, batch_size, shuffle_rows=False, pick_col_size=800):
st = time.time()
val_result = train_targets_scored.loc[:,_y_i_train.columns].copy()
val_result.loc[:, :] = 0
sub_result = sample_submission.loc[:, _y_i_train.columns].copy()
sub_result.loc[:, :] = 0
all_cols = _X_i_train.columns.values
for h, seed in enumerate(seeds):
seed_everything(seed)
 for i,(train_idx, valid_idx)in enumerate(MultilabelStratifiedKFold(n_splits=splits, random_state=seed, shuffle=True)
.split(_y_i_train, _y_i_train)) :
print(f'Fold {i+1}')
 if shuffle_rows:
seed_everything(seed + i)
 target_cols = np.random.choice(all_cols, size=pick_col_size)
 seed_everything(seed)
 else:
target_cols = all_cols
_X_train = _X_i_train.loc[:,target_cols].values[train_idx]
_y_train = _y_i_train.values[train_idx]
_X_valid = _X_i_train.loc[:,target_cols].values[valid_idx]
_y_valid = _y_i_train.values[valid_idx]
model = model_func(len(target_cols), len(_y_i_train.columns))
model.fit(_X_train, _y_train,
validation_data=(_X_valid, _y_valid),
epochs=epochs, batch_size=batch_size,
callbacks=[
ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=3, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'{name}_{seed}_{i}.hdf5', monitor = 'val_logloss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 , EarlyStopping(monitor = 'val_logloss', min_delta = 1e-4, patience = 5, mode = 'min', baseline = None, restore_best_weights = True)
 ], verbose=2)
 model.load_weights(f'{name}_{seed}_{i}.hdf5')
 val_result.iloc[y_train.iloc[valid_idx,:].index.values, :] += model.predict(_X_valid)
 sub_result.loc[test_trt_index, :] += model.predict(_X_i_test.loc[test_trt_index, target_cols])
 print('')
 tmp_result = val_result.copy()
tmp_result.iloc[:,1:] = val_result.iloc[:,1:] /(h + 1)
 print(f' ---- seed:{seed}, ensemble:{h + 1}')
 _ = show_metrics(tmp_result, show_each=False)
 val_result.iloc[:,1:] = val_result.iloc[:,1:] / len(seeds)
 metrics = show_metrics(val_result)
 sub_result.iloc[:, :] = sub_result.iloc[:, :] /(len(seeds)* splits)
 print(f' elapsed: {time.time() - st}')
 return sub_result, val_result, metrics
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121<define_variables>
X_stacking_train = pd.concat([nn_valid_preds, rnn_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = pd.concat([X_stacking_train, tab_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = pd.concat([X_stacking_train, nn2_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = pd.concat([X_stacking_train, snn_valid_preds], axis=1, ignore_index=True)
 X_stacking_train = X_stacking_train[train_trt_index]
X_stacking_test = pd.concat([nn_result.iloc[:, 1:], rnn_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = pd.concat([X_stacking_test, tab_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = pd.concat([X_stacking_test, nn2_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = pd.concat([X_stacking_test, snn_result.iloc[:, 1:]], axis=1, ignore_index=True)
 X_stacking_test = X_stacking_test
img_size = 256
batch_size = 256<define_search_model>
def create_stacking_model(input_dim, output_dim):
print(f'the input dim is {input_dim}')
 model = M.Sequential()
model.add(L.Input(input_dim))
model.add(L.BatchNormalization())
 model.add(L.Dropout(0.3))
model.add(WeightNormalization(L.Dense(input_dim, activation='elu')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(input_dim, activation='swish')))
 model.add(L.BatchNormalization())
 model.add(L.Dropout(0.5))
model.add(WeightNormalization(L.Dense(output_dim, activation='sigmoid', bias_initializer=output_bias)))
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=1e-3), sync_period=10),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING), metrics=logloss)
 return model
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<define_variables>
if RUN_STACKING:
st_re_result, st_re_valid_preds, st_re_metrics = re_train_and_predict('ST_NN', create_stacking_model,
X_stacking_train, y_train, X_stacking_test,
STK_SEEDS, STK_SPLITS, STK_EPOCHS, STK_BATCH_SIZE )
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
with open('nn_valid_preds.pkl', 'wb')as p:
pickle.dump(nn_valid_preds , p)
 with open('nn2_valid_preds.pkl', 'wb')as p:
pickle.dump(nn2_valid_preds , p)
 with open('rnn_valid_preds.pkl', 'wb')as p:
pickle.dump(rnn_valid_preds , p)
 with open('tab_valid_preds.pkl', 'wb')as p:
pickle.dump(tab_valid_preds , p)
 with open('st_re_valid_preds.pkl', 'wb')as p:
pickle.dump(st_re_valid_preds , p)
 with open('snn_valid_preds.pkl', 'wb')as p:
pickle.dump(snn_valid_preds , p )
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]<define_variables>
ratio = [0.05, 0.05, 0.1, 0.25, 0.35, 0.2]
ens_valid_preds = nn_valid_preds * ratio[0] + nn2_valid_preds * ratio[1] + rnn_valid_preds * ratio[2] + tab_valid_preds * ratio[3] + \
st_re_valid_preds * ratio[4] + snn_valid_preds * ratio[5]
_ = show_metrics(ens_valid_preds, show_each=False )
<create_dataframe><EOS> 
ens_result = nn_result.iloc[:, 1:] * ratio[0] + nn2_result.iloc[:, 1:] * ratio[1] + rnn_result.iloc[:, 1:] * ratio[2] + \
tab_result.iloc[:, 1:] * ratio[3] + st_re_result * ratio[4] + snn_result.iloc[:, 1:] * ratio[5]
ens_result = pd.concat([nn_result.iloc[:,0], ens_result], axis=1)
 ens_result.to_csv('submission.csv', index=False )