Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<categorify>
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import os
import random
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )<feature_engineering>
print(f'Numpy: {np.__version__}')
 print(f'Pandas: {pd.__version__}')
 print(f'Matplotlib: {matplotlib.__version__}')
 print(f'Seaborn: {sns.__version__}' )
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )<drop_column>
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 print('Train Sales Data')
 train.head(2 )
features_to_drop = ["sig_id", "cp_type"]
try:
data_all.drop(features_to_drop, axis = 1, inplace = True)
 except:
pass
try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )<import_modules>
train.isna().sum()
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )<load_from_csv>
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 print('Items Data')
 items.head(2 )
SEED = 42
FOLDS = 5
def create_fold() :
train_feats = pd.read_csv('.. /input/lish-moa/train_features.csv')
 scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 drug = pd.read_csv('/kaggle/input/lish-moa/train_drug.csv')
 targets = scored.columns[1:]
scored = scored.merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 scored = scored.loc[ctl_vehicle_train]
return scored
scored = create_fold()
scored<drop_column>
items.isna().sum()
scored = scored.drop(["sig_id", "drug_id"], axis=1 ).reset_index(drop=True)
 train_df = train_df.reset_index(drop=True)
 scored<define_variables>
items.nunique()
if encoding == "lb":
categoricals = [train_df.columns.get_loc("cp_time"), train_df.columns.get_loc("cp_dose")]
else:
categoricals = [train_df.columns.get_loc(x)for x in train_df.columns if "cp_" in x]
categoricals<choose_model_class>
it_cat = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 print('Item Categories Data')
 it_cat.head(2 )
MAX_EPOCH = 400
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay=1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 5,
cat_idxs = categoricals 
)
 scores_auc_all = []
test_cv_preds = []
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for i in range(FOLDS):
print(b_,"FOLDS: ", r_, i + 1)
 print(g_, '*' * 60, c_)
 idx = scored["fold"] == i
not_idx = scored["fold"] != i
scored_df = scored.drop("fold", axis=1)
 X_train, y_train = train_df.values[not_idx, :], scored_df.values[not_idx, :]
X_val, y_val = train_df.values[idx, :], scored_df.values[idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 30,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<compute_test_metric>
it_cat.isna().sum()
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )<save_to_csv>
it_cat.nunique()
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()<import_modules>
shop = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 print("Shops Data")
 shop.head(2 )
import os
import random
import time
import json
import warnings
import numpy as np
import pandas as pd
from sklearn.utils.class_weight import compute_class_weight
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.base import BaseEstimator
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, QuantileTransformer
from sklearn.metrics import fbeta_score
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import regularizers
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
import matplotlib.pyplot as plt
from xgboost import XGBClassifier<define_variables>
shop.isna().sum()
sys.path += ['/kaggle/input/iterstrat', '/kaggle/input/tabnet'] 
<define_variables>
shop.nunique()
def set_random_seeds(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 set_random_seeds(43 )<load_from_csv>
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 print('Test Data')
 test.head(2 )
X_full = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 Y_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 Y_nonscored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 drug = pd.read_csv('/kaggle/input/lish-moa/train_drug.csv')
 X_test_full = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sig_id = X_test_full['sig_id'].values.reshape(( -1, 1))<define_variables>
test.isna().sum()
cat_cols = ['cp_type', 'cp_time', 'cp_dose']
numerical_cols = [c for c in X_full.columns if c not in ['sig_id'] + cat_cols]
label_cols = [c for c in Y_scored.columns if c != 'sig_id']
num_features = X_full.shape[1] - 1
num_labels = Y_scored.shape[1] - 1
num_nonscored_labels = Y_nonscored.shape[1] - 1<compute_train_metric>
test.nunique()
def logloss(Y_true, Y_preds, label_smoothing=0):
Y_true = Y_true.astype(np.float32)
 return tf.reduce_mean(keras.losses.binary_crossentropy(Y_true, Y_preds, label_smoothing=label_smoothing)).numpy()<filter>
train.nunique()
def add_outputs(Y, Y_preds, X=None, idx_list=None, col=None):
if X is not None:
if col is None:
Y.loc[X['cp_type'] == 'trt_cp', :] += Y_preds
else:
Y.loc[X['cp_type'] == 'trt_cp', Y.columns[col]] += Y_preds
elif idx_list is not None:
if col is None:
Y.iloc[idx_list, :] += Y_preds
else:
Y.iloc[idx_list, col] += Y_preds
else:
raise Exception("Must provide X or idx_list" )<categorify>
sub = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv')
 sub.head(2 )
def balance_class_weights(Y_true, class_weights=(0.7, 1.6)) :
n_labels = Y_true.shape[1]
weights = np.empty([n_labels, 2])
 for i in range(n_labels):
if class_weights == 'balanced':
total = len(Y_true[:, i])
 pos = np.sum(Y_true[:, i])
 neg = total - pos
weight_for_0 =(1 / neg)*(total)/2.0
weight_for_1 =(1 / pos)*(total)/2.0
weights[i] = [weight_for_0, weight_for_1]
else:
weights[i] = np.array(class_weights)
 return weights
def weighted_binary_crossentropy(weights, label_smoothing=0):
def weighted_loss(Y_true, Y_pred):
bce = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 return K.mean(
(weights[:, 0]**(1 - Y_true)) *(weights[:, 1]**(Y_true)) * bce(Y_true, Y_pred),
axis=-1 
)
 return weighted_loss<categorify>
train[train.item_cnt_day<0]
def preprocess(df, targets=None):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df = df.drop(columns='sig_id')
 if targets is not None:
targets = targets.copy()
targets = targets[df['cp_type'] == 0].reset_index(drop=True)
 df = df[df['cp_type'] == 0].reset_index(drop=True)
 if targets is None:
return df
return df, targets<merge>
return_per_shop = train[train.item_cnt_day<0].groupby('shop_id')['item_cnt_day'].sum()
return_per_shop
X_train, Y_train = preprocess(X_full, Y_scored)
 X_test = preprocess(X_test_full)
 Y_train_nonscored = Y_nonscored.drop(columns='sig_id')[X_full['cp_type'] == 'trt_cp'].reset_index(drop=True)
 initial_bias = -np.log(Y_train.mean(axis=0 ).values)
 nonscored_initial_bias = -np.log(Y_train_nonscored.mean(axis=0 ).values + 1e-6)
 Y_train_with_drugs = Y_train.merge(drug, on='sig_id', how='left')
 Y_train = Y_train.drop(columns='sig_id')
 vc = Y_train_with_drugs.drug_id.value_counts()
vc1 = vc.loc[(vc == 6)|(vc == 12)|(vc == 18)].index.sort_values()
vc2 = vc.loc[(vc != 6)&(vc != 12)&(vc != 18)].index.sort_values()<feature_engineering>
return_per_shop.sort_values(ascending=True ).iloc[:10]
def select_features(X):
return X.iloc[:, [
1, 2, 3, 4, 5, 6, 7, 9, 11, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31,
32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46,
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 58, 59, 60,
61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88,
89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101,
102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 120, 121, 122, 123, 124, 125, 126, 127, 128,
129, 130, 131, 132, 133, 136, 137, 138, 139, 140, 141, 142, 143,
144, 145, 146, 147, 149, 150, 151, 152, 153, 154, 155, 156, 157,
158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170,
171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183,
184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 197,
198, 199, 200, 202, 203, 204, 205, 206, 208, 209, 210, 211, 212,
213, 214, 215, 216, 217, 218, 219, 220, 221, 223, 224, 225, 226,
227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253,
254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266,
267, 268, 269, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280,
281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294,
295, 296, 298, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309,
310, 311, 312, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336,
337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 367, 368, 369, 370, 371, 374, 375, 376, 377,
378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 390, 391,
392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404,
405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 416, 417, 418,
419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431,
432, 434, 435, 436, 437, 438, 439, 440, 442, 443, 444, 445, 446,
447, 448, 449, 450, 453, 454, 456, 457, 458, 459, 460, 461, 462,
463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475,
476, 477, 478, 479, 481, 482, 483, 484, 485, 486, 487, 488, 489,
490, 491, 492, 493, 494, 495, 496, 498, 500, 501, 502, 503, 505,
506, 507, 509, 510, 511, 512, 513, 514, 515, 518, 519, 520, 521,
522, 523, 524, 525, 526, 527, 528, 530, 531, 532, 534, 535, 536,
538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 549, 550, 551,
552, 554, 557, 559, 560, 561, 562, 565, 566, 567, 568, 569, 570,
571, 572, 573, 574, 575, 577, 578, 580, 581, 582, 583, 584, 585,
586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 599,
600, 601, 602, 606, 607, 608, 609, 611, 612, 613, 615, 616, 617,
618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630,
631, 632, 633, 634, 635, 636, 637, 638, 639, 641, 642, 643, 644,
645, 646, 647, 648, 649, 650, 651, 652, 654, 655, 656, 658, 659,
660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672,
673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685,
686, 687, 688, 689, 691, 692, 693, 694, 695, 696, 697, 699, 700,
701, 702, 704, 705, 707, 708, 709, 710, 711, 713, 714, 716, 717,
718, 720, 721, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732,
733, 734, 735, 737, 738, 739, 740, 742, 743, 744, 745, 746, 747,
748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 759, 760, 761,
762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774,
775, 776, 777, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788,
789, 790, 792, 793, 794, 795, 796, 797, 798, 800, 801, 802, 803,
804, 805, 806, 808, 809, 811, 813, 814, 815, 816, 817, 818, 819,
821, 822, 823, 825, 826, 827, 828, 829, 830, 831, 832, 834, 835,
837, 838, 839, 840, 841, 842, 845, 846, 847, 848, 850, 851, 852,
854, 855, 856, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869,
870, 871, 872, 873, 874
]].values<define_variables>
return_per_shop.sort_values(ascending=False ).iloc[:10]
gene_cols = [c for c in X_test.columns if c.startswith('g-')]
cell_cols = [c for c in X_test.columns if c.startswith('c-')]<normalization>
return_per_item = train[train.item_cnt_day<0].groupby('item_id')['item_cnt_day'].sum()
return_per_item
qtransform = joblib.load('/kaggle/input/qtransform/qtransform.joblib')
 X_test_selected = select_features(X_test)
 X_test_qtrans = np.concatenate([X_test_selected[:, :2], qtransform.transform(X_test_selected[:, 2:])], axis=1)
 pca_qtrans_2 = joblib.load('/kaggle/input/qtransform/pca_qtrans.joblib' )<load_pretrained>
return_per_item.sort_values(ascending=True ).iloc[:10]
std_scaler_2 = joblib.load('/kaggle/input/qtransform/std_scaler_2.joblib')
 gene_pca_2 = joblib.load('/kaggle/input/qtransform/gene_pca_2.joblib')
 cell_pca_2 = joblib.load('/kaggle/input/qtransform/cell_pca_2.joblib' )<load_pretrained>
return_per_item.sort_values(ascending=False ).iloc[:10]
kmeans_g = joblib.load('/kaggle/input/qtransform/kmeans_g.joblib')
 kmeans_c = joblib.load('/kaggle/input/qtransform/kmeans_c.joblib' )<normalization>
returns_monthly = train[train.item_cnt_day<0].groupby('date_block_num')['item_cnt_day'].sum()
returns_monthly
def preprocess_nn(X):
X_pca = pca_qtrans_2.transform(X)
 return np.concatenate([X, X_pca], axis=1 )<compute_test_metric>
no_sold = train[train.item_cnt_day==0.0]
no_sold
X_test_nn = preprocess_nn(X_test_qtrans )<choose_model_class>
sold_per_month = train.groupby('date_block_num')['item_cnt_day'].sum()
class MultiLabelNN(BaseEstimator):
def __init__(self,
n_input_features=num_features,
n_output_labels=num_labels,
n_hidden_units=2048):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.n_hidden_units = n_hidden_units
self._model_nn = None
def _create_nn_model(self, loss_fn=None, loss_fn_aux=None):
output_bias = keras.initializers.Constant(initial_bias)
 aux_output_bias = keras.initializers.Constant(nonscored_initial_bias)
 inp = keras.layers.Input(self.n_input_features)
 gauss_noise = keras.layers.GaussianNoise(0.001 )(inp)
 norm_1 = keras.layers.BatchNormalization()(gauss_noise)
 dense_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(norm_1)
 norm_2 = keras.layers.BatchNormalization()(dense_1)
 dropout_1 = keras.layers.Dropout(0.4 )(norm_2)
 dense_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_1)
 norm_3 = keras.layers.BatchNormalization()(dense_2)
 dropout_2 = keras.layers.Dropout(0.4 )(norm_3)
 dropout_3 = keras.layers.Dropout(0.5 )(norm_2)
 dense_3 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='relu',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_3)
 norm_4 = keras.layers.BatchNormalization()(dense_3)
 dropout_4 = keras.layers.Dropout(0.4 )(norm_4)
 output_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
), name='main' )(dropout_4)
 output_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 Y_train_nonscored.shape[1],
activation='sigmoid',
bias_initializer=aux_output_bias 
), name='aux' )(dropout_2)
 model_nn = keras.models.Model(inputs=inp, outputs=[output_1, output_2])
 model_nn.compile(
 loss=[
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn is None else loss_fn,
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn_aux is None else loss_fn_aux
],
optimizer=keras.optimizers.Nadam(learning_rate=0.01),
metrics=['binary_crossentropy'],
loss_weights=[0.9, 0.1] 
)
 return model_nn
def fit(self, X, Y, X_valid, Y_valid, Y_aux, Y_aux_valid, max_epochs=100, save_weights=True):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 Y_aux = tf.cast(Y_aux, tf.float32)
 Y_aux_valid = tf.cast(Y_aux_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.7, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 class_weights_aux = balance_class_weights(Y_aux.numpy() , class_weights=(0.7, 1.4))
loss_fn_aux = weighted_binary_crossentropy(class_weights_aux, label_smoothing=1e-3)
 self._model_nn = self._create_nn_model(loss_fn=loss_fn, loss_fn_aux=loss_fn_aux)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=10,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_nn/weights_{repeat}_{fold}.h5',
monitor='val_main_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self._model_nn.fit(
 X, [Y, Y_aux],
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, [Y_valid, Y_aux_valid]),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self._model_nn.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)[0]
return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self._model_nn.save_weights('model_nn/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self._model_nn is None:
self._model_nn = self._create_nn_model()
if pretrained:
base_path = '/kaggle/input/model-nn-drug-cv/'
else:
base_path = '/kaggle/working/'
self._model_nn.load_weights(base_path + 'model_nn/weights_{}_{}.h5'.format(repeat, fold))<predict_on_test>
tmp_data = train.groupby('item_id', as_index=False)['date_block_num'].min()
tmp_data
n_repeats = 5
n_folds = 7
Y_test_preds_nn = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_nn = MultiLabelNN(n_input_features=X_test_nn.shape[1])
 model_nn.load_weights(repeat, fold)
 test_preds = model_nn.predict(X_test_nn)[0]
add_outputs(Y_test_preds_nn, test_preds, X=X_test_full)
 del model_nn
Y_test_preds_nn.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )<normalization>
tmp_data[tmp_data.item_id.isin(test_items)].sort_values(by='date_block_num', ascending=False )
def preprocess_tabnet(X, X_qtrans):
X_pca = pca_qtrans_2.transform(X_qtrans)
 g_std = X[gene_cols].std(axis=1 ).values.reshape(-1, 1)
 g_kurt = X[gene_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 g_skew = X[gene_cols].skew(axis=1 ).values.reshape(-1, 1)
 c_std = X[cell_cols].std(axis=1 ).values.reshape(-1, 1)
 c_kurt = X[cell_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 c_skew = X[cell_cols].skew(axis=1 ).values.reshape(-1, 1)
 X_std = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 return np.concatenate([
X_qtrans, X_pca,
g_std, g_kurt, g_skew,
c_std, c_kurt, c_skew,
], axis=1 )<concatenate>
train[train['item_cnt_day']>=1000]
X_test_tabnet = preprocess_tabnet(X_test, X_test_qtrans )<choose_model_class>
items[items['item_id'].isin([20949, 11373])]['item_name'].values
class MultiLabelTabNet(BaseEstimator):
def __init__(self, n_input_features, n_output_labels):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, loss_fn=None, output_bias=None):
if output_bias is not None:
output_bias = keras.initializers.Constant(output_bias)
 clf = StackedTabNetClassifier(
 feature_columns=None,
num_classes=1024,
num_layers=1,
feature_dim=2048,
output_dim=1024,
num_features=self.n_input_features,
num_decision_steps=1,
relaxation_factor=1.5,
sparsity_coefficient=1e-5,
norm_type='batch',
num_groups=-1,
output_activation='relu' 
)
 model = keras.models.Sequential([
keras.layers.InputLayer(self.n_input_features),
keras.layers.Dropout(0.5),
keras.layers.GaussianNoise(0.001),
clf,
keras.layers.Dropout(0.7),
tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
))
])
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(weight_decay=2.6e-5),
metrics=['binary_crossentropy'] 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=200):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 15.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 self.estimator_ = self._create_model(loss_fn=loss_fn, output_bias=initial_bias)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss', min_delta=1e-5,
patience=3, verbose=1, factor=0.1, min_lr=1e-7 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5, patience=10, restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_tabnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=[early_stop_cb, learning_rate_cb, checkpoint_cb],
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_tabnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model()
if pretrained:
base_path = '/kaggle/input/model-tabnet-drug-cv/'
else:
base_path = './'
self.estimator_.load_weights(base_path + f'model_tabnet/weights_{repeat}_{fold}.h5' )<prepare_x_and_y>
train[train['item_price']>=100000]
n_repeats = 5
n_folds = 7
Y_test_preds_tabnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_tabnet = MultiLabelTabNet(
 n_input_features=X_test_tabnet.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_tabnet.load_weights(repeat, fold)
 test_preds = model_tabnet.predict(X_test_tabnet)
 add_outputs(Y_test_preds_tabnet, test_preds, X=X_test_full)
 del model_tabnet
Y_test_preds_tabnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )<normalization>
items[items['item_id']==6066]
def preprocess_resnet(X):
X_scaled = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 X_genes = gene_pca_2.transform(X_scaled[gene_cols])
 X_cells = cell_pca_2.transform(X_scaled[cell_cols])
 return select_features(X), np.concatenate([X_genes, X_cells], axis=1 )<statistical_test>
train[train['item_id']==6066]
X_test_resnet_1, X_test_resnet_2 = preprocess_resnet(X_test )<choose_model_class>
train = train[train['item_cnt_day']<1000]
train = train[train['item_price']<100000]
train.reset_index(drop=True, inplace=True )
class MultiLabelResNet(BaseEstimator):
def __init__(self, n_input_1, n_input_2, n_output_labels):
self.n_input_1 = n_input_1
self.n_input_2 = n_input_2
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, initial_bias, loss_fn=None):
output_bias = keras.initializers.Constant(initial_bias)
 inp_1 = keras.layers.Input(self.n_input_1, name='raw_inp')
 inp_2 = keras.layers.Input(self.n_input_2, name='pca_inp')
 gauss_2 = keras.layers.GaussianNoise(1e-4 )(inp_2)
 head_1 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(keras.layers.Dense(512, activation='elu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='elu'))
], name='head1')
 seq_1 = head_1(inp_1)
 seq_1_inp_concat = keras.layers.Concatenate()([gauss_2, seq_1])
 head_2 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "elu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "elu"))
], name='head2')
 seq_2 = head_2(seq_1_inp_concat)
 seq_1_seq_2_avg = keras.layers.Average()([seq_1, seq_2])
 head_3 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(206, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(self.n_output_labels, bias_initializer=output_bias, activation='sigmoid'))
], name='head3')
 output = head_3(seq_1_seq_2_avg)
 model = keras.models.Model(inputs=[inp_1, inp_2], outputs=output)
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(learning_rate=0.02, weight_decay=1e-5),
metrics=['binary_crossentropy'], 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=100, save_weights=True, initial_bias=initial_bias, transfer_model=None):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(initial_bias, loss_fn=loss_fn)
 if transfer_model is not None:
self.copy_weights(transfer_model)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=15,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_resnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model(initial_bias)
 if pretrained:
base_path = '/kaggle/input/model-resnet-drug-cv/'
else:
base_path = './'
self.estimator_.load_weights(base_path + 'model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def copy_weights(self, from_model):
for to_layer, from_layer in zip(self.estimator_.layers[:-1], from_model.estimator_.layers[:-1]):
to_layer.set_weights(from_layer.get_weights())
 for to_layer, from_layer in zip(self.estimator_.layers[-1].layers[:-1], from_model.estimator_.layers[-1].layers[:-1]):
to_layer.set_weights(from_layer.get_weights() )<create_dataframe>
import holidays
n_repeats = 7
n_folds = 7
Y_test_preds_resnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 K.clear_session()
for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
model_resnet = MultiLabelResNet(
 n_input_1=X_test_resnet_1.shape[1],
n_input_2=X_test_resnet_2.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_resnet.load_weights(repeat, fold)
 test_preds = model_resnet.predict([X_test_resnet_1, X_test_resnet_2])
 add_outputs(Y_test_preds_resnet, test_preds, X=X_test_full)
 del model_resnet
Y_test_preds_resnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )<define_search_space>
ru_holidays = holidays.Russia()
weights = [0.35, 0.40, 0.25]
print(sum(weights))
assert(abs(sum(weights)- 1)< 1e-6)
 final_preds_drug_cv =(
 Y_test_preds_nn * weights[0] +
Y_test_preds_tabnet * weights[1] +
Y_test_preds_resnet * weights[2]
)<normalization>
train['new_date'] = train['date'].apply(lambda x: x.split('.')[1]+'/'+x.split('.')[0]+'/'+x.split('.')[2] )
def preprocess_nn(X):
X_pca = pca_qtrans_2.transform(X)
 return np.concatenate([X, X_pca], axis=1 )<compute_test_metric>
train['is_holiday'] = train['new_date'].apply(lambda x: x in ru_holidays )
X_test_nn = preprocess_nn(X_test_qtrans )<choose_model_class>
train[train['date_block_num']==0]['new_date'].nunique()
class MultiLabelNN(BaseEstimator):
def __init__(self,
n_input_features=num_features,
n_output_labels=num_labels,
n_hidden_units=2048):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.n_hidden_units = n_hidden_units
self._model_nn = None
def _create_nn_model(self, loss_fn=None, loss_fn_aux=None):
output_bias = keras.initializers.Constant(initial_bias)
 aux_output_bias = keras.initializers.Constant(nonscored_initial_bias)
 inp = keras.layers.Input(self.n_input_features)
 gauss_noise = keras.layers.GaussianNoise(0.001 )(inp)
 norm_1 = keras.layers.BatchNormalization()(gauss_noise)
 dense_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(norm_1)
 norm_2 = keras.layers.BatchNormalization()(dense_1)
 dropout_1 = keras.layers.Dropout(0.4 )(norm_2)
 dense_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_1)
 norm_3 = keras.layers.BatchNormalization()(dense_2)
 dropout_2 = keras.layers.Dropout(0.4 )(norm_3)
 dropout_3 = keras.layers.Dropout(0.5 )(norm_2)
 dense_3 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='relu',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_3)
 norm_4 = keras.layers.BatchNormalization()(dense_3)
 dropout_4 = keras.layers.Dropout(0.4 )(norm_4)
 output_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
), name='main' )(dropout_4)
 output_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 Y_train_nonscored.shape[1],
activation='sigmoid',
bias_initializer=aux_output_bias 
), name='aux' )(dropout_2)
 model_nn = keras.models.Model(inputs=inp, outputs=[output_1, output_2])
 model_nn.compile(
 loss=[
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn is None else loss_fn,
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn_aux is None else loss_fn_aux
],
optimizer=keras.optimizers.Nadam(learning_rate=0.01),
metrics=['binary_crossentropy'],
loss_weights=[0.9, 0.1] 
)
 return model_nn
def fit(self, X, Y, X_valid, Y_valid, Y_aux, Y_aux_valid, max_epochs=100, save_weights=True):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 Y_aux = tf.cast(Y_aux, tf.float32)
 Y_aux_valid = tf.cast(Y_aux_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.7, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 class_weights_aux = balance_class_weights(Y_aux.numpy() , class_weights=(0.7, 1.4))
loss_fn_aux = weighted_binary_crossentropy(class_weights_aux, label_smoothing=1e-3)
 self._model_nn = self._create_nn_model(loss_fn=loss_fn, loss_fn_aux=loss_fn_aux)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=10,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_nn/weights_{repeat}_{fold}.h5',
monitor='val_main_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self._model_nn.fit(
 X, [Y, Y_aux],
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, [Y_valid, Y_aux_valid]),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self._model_nn.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)[0]
return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self._model_nn.save_weights('model_nn/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self._model_nn is None:
self._model_nn = self._create_nn_model()
if pretrained:
base_path = '/kaggle/input/model-nn/'
else:
base_path = './working/'
self._model_nn.load_weights(base_path + 'model_nn/weights_{}_{}.h5'.format(repeat, fold))<predict_on_test>
tmp_data = train.groupby(['date_block_num', 'new_date'], as_index=False)['is_holiday'].sum()
tmp_data['is_holiday'] = tmp_data['is_holiday']>0
n_holidays = tmp_data.groupby(['date_block_num'])['is_holiday'].sum()
n_holidays
n_repeats = 5
n_folds = 7
Y_test_preds_nn = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_nn = MultiLabelNN(n_input_features=X_test_nn.shape[1])
 model_nn.load_weights(repeat, fold)
 test_preds = model_nn.predict(X_test_nn)[0]
add_outputs(Y_test_preds_nn, test_preds, X=X_test_full)
 del model_nn
Y_test_preds_nn.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )<normalization>
train['new_date'] = pd.to_datetime(train['new_date'] )
def preprocess_tabnet(X, X_qtrans):
X_pca = pca_qtrans_2.transform(X_qtrans)
 g_std = X[gene_cols].std(axis=1 ).values.reshape(-1, 1)
 g_kurt = X[gene_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 g_skew = X[gene_cols].skew(axis=1 ).values.reshape(-1, 1)
 c_std = X[cell_cols].std(axis=1 ).values.reshape(-1, 1)
 c_kurt = X[cell_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 c_skew = X[cell_cols].skew(axis=1 ).values.reshape(-1, 1)
 X_std = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 g_cluster = kmeans_g.predict(X_std[gene_cols] ).reshape(-1, 1)
 c_cluster = kmeans_c.predict(X_std[cell_cols] ).reshape(-1, 1)
 return np.concatenate([
X_qtrans, X_pca,
g_std, g_kurt, g_skew,
c_std, c_kurt, c_skew,
g_cluster, c_cluster
], axis=1 )<concatenate>
train['weekend'] = train['new_date'].apply(lambda x : x.weekday() in [5, 6] )
X_test_tabnet = preprocess_tabnet(X_test, X_test_qtrans )<choose_model_class>
tmp_data = train.groupby(['date_block_num', 'date'], as_index=False)['weekend'].sum()
tmp_data['weekend'] = tmp_data['weekend']>0
n_weekends = tmp_data.groupby(['date_block_num'])['weekend'].sum()
n_weekends
class MultiLabelTabNet(BaseEstimator):
def __init__(self, n_input_features, n_output_labels):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, loss_fn=None, output_bias=None):
if output_bias is not None:
output_bias = keras.initializers.Constant(output_bias)
 clf = StackedTabNetClassifier(
 feature_columns=None,
num_classes=1024,
num_layers=1,
feature_dim=2048,
output_dim=1024,
num_features=self.n_input_features,
num_decision_steps=1,
relaxation_factor=1.5,
sparsity_coefficient=1e-5,
num_groups=-1,
output_activation='relu' 
)
 model = keras.models.Sequential([
keras.layers.InputLayer(self.n_input_features),
keras.layers.Dropout(0.5),
keras.layers.GaussianNoise(0.001),
clf,
keras.layers.Dropout(0.7),
tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
))
])
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(weight_decay=2.6e-5),
metrics=['binary_crossentropy'] 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=200):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 15.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(loss_fn=loss_fn, output_bias=initial_bias)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss', min_delta=1e-5,
patience=3, verbose=1, factor=0.1, min_lr=1e-7 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5, patience=10, restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_tabnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=[early_stop_cb, learning_rate_cb, checkpoint_cb],
shuffle=True,
verbose=2 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_tabnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model()
if pretrained:
base_path = '/kaggle/input/model-tabnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + f'model_tabnet/weights_{repeat}_{fold}.h5' )<prepare_x_and_y>
train['money'] = train['item_price'] * train['item_cnt_day']
n_repeats = 5
n_folds = 7
Y_test_preds_tabnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
model_tabnet = MultiLabelTabNet(
 n_input_features=X_test_tabnet.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_tabnet.load_weights(repeat, fold)
 test_preds = model_tabnet.predict(X_test_tabnet)
 add_outputs(Y_test_preds_tabnet, test_preds, X=X_test_full)
 del model_tabnet
Y_test_preds_tabnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )<normalization>
train.groupby(['date_block_num', 'shop_id'], as_index=False)['money'].sum()
def preprocess_resnet(X):
X_scaled = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 X_genes = gene_pca_2.transform(X_scaled[gene_cols])
 X_cells = cell_pca_2.transform(X_scaled[cell_cols])
 return select_features(X), np.concatenate([X_genes, X_cells], axis=1 )<statistical_test>
train['month'] = train['date'].apply(lambda x: x.split('.')[1])
 train
X_test_resnet_1, X_test_resnet_2 = preprocess_resnet(X_test )<choose_model_class>
train.loc[train.shop_id == 0, "shop_id"] = 57
test.loc[test.shop_id == 0 , "shop_id"] = 57
train.loc[train.shop_id == 1, "shop_id"] = 58
test.loc[test.shop_id == 1 , "shop_id"] = 58
train.loc[train.shop_id == 11, "shop_id"] = 10
test.loc[test.shop_id == 11, "shop_id"] = 10
train.loc[train.shop_id == 40, "shop_id"] = 39
test.loc[test.shop_id == 40, "shop_id"] = 39
class MultiLabelResNet(BaseEstimator):
def __init__(self, n_input_1, n_input_2, n_output_labels):
self.n_input_1 = n_input_1
self.n_input_2 = n_input_2
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, initial_bias, loss_fn=None):
output_bias = keras.initializers.Constant(initial_bias)
 inp_1 = keras.layers.Input(self.n_input_1, name='raw_inp')
 inp_2 = keras.layers.Input(self.n_input_2, name='pca_inp')
 gauss_2 = keras.layers.GaussianNoise(1e-4 )(inp_2)
 head_1 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(keras.layers.Dense(512, activation='elu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='elu'))
], name='head1')
 seq_1 = head_1(inp_1)
 seq_1_inp_concat = keras.layers.Concatenate()([gauss_2, seq_1])
 head_2 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "elu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "elu"))
], name='head2')
 seq_2 = head_2(seq_1_inp_concat)
 seq_1_seq_2_avg = keras.layers.Average()([seq_1, seq_2])
 head_3 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(206, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(self.n_output_labels, bias_initializer=output_bias, activation='sigmoid'))
], name='head3')
 output = head_3(seq_1_seq_2_avg)
 model = keras.models.Model(inputs=[inp_1, inp_2], outputs=output)
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(learning_rate=0.02, weight_decay=1e-5),
metrics=['binary_crossentropy'], 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=100, save_weights=True, initial_bias=initial_bias, transfer_model=None):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(initial_bias, loss_fn=loss_fn)
 if transfer_model is not None:
self.copy_weights(transfer_model)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=15,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_resnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model(initial_bias)
 if pretrained:
base_path = '/kaggle/input/model-resnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + 'model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def copy_weights(self, from_model):
for to_layer, from_layer in zip(self.estimator_.layers[:-1], from_model.estimator_.layers[:-1]):
to_layer.set_weights(from_layer.get_weights())
 for to_layer, from_layer in zip(self.estimator_.layers[-1].layers[:-1], from_model.estimator_.layers[-1].layers[:-1]):
to_layer.set_weights(from_layer.get_weights() )<create_dataframe>
X_train = train.groupby(['date_block_num', 'shop_id', 'item_id'], as_index=False)['item_cnt_day'].sum()
X_train
n_repeats = 7
n_folds = 7
Y_test_preds_resnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 for repeat in range(n_repeats):
for fold in range(n_folds):
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
model_resnet = MultiLabelResNet(
 n_input_1=X_test_resnet_1.shape[1],
n_input_2=X_test_resnet_2.shape[1],
n_output_labels=Y_train.shape[1] 
)
 model_resnet.load_weights(repeat, fold)
 test_preds = model_resnet.predict([X_test_resnet_1, X_test_resnet_2])
 add_outputs(Y_test_preds_resnet, test_preds, X=X_test_full)
 del model_resnet
Y_test_preds_resnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats )<define_search_space>
X_train['Weekends'] = X_train['date_block_num'].map(n_weekends)
 X_train['Holidays'] = X_train['date_block_num'].map(n_holidays)
 X_train
weights = [0.22, 0.43, 0.35]
print(sum(weights))
assert(abs(sum(weights)- 1)< 1e-6)
 final_preds_raw_cv =(
 Y_test_preds_nn * weights[0] +
Y_test_preds_tabnet * weights[1] +
Y_test_preds_resnet * weights[2]
)<save_to_csv>
X_train['month'] = X_train['date_block_num'].map(train.groupby(['date_block_num'])['month'].unique())
 X_train['month'] = X_train['month'].apply(lambda x : int(x[0]))
final_preds = 0.4 * final_preds_drug_cv + 0.6 * final_preds_raw_cv
final_preds.loc[X_test_full['cp_type'] != 'trt_cp', :] = 0
final_preds = np.concatenate([sig_id, final_preds], axis=1)
 final_preds = pd.DataFrame(final_preds, columns=Y_scored.columns)
 final_preds.to_csv('submission.csv', index=False )<import_modules>
shop.loc[ shop.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shop["city"] = shop.shop_name.str.split(" " ).map(lambda x: x[0])
 shop["type"] = shop.shop_name.str.split(" " ).map(lambda x: x[1])
 shop.loc[shop.city == "!Якутск", "city"] = "Якутск"
import numpy as np
import pandas as pd<load_from_csv>
shop.loc[~shop.type.isin(['ТЦ', 'ТРЦ', 'ТК', 'ТРК', 'MTPЦ']),"type"] = 'other'
shop
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_ids = list(submission['sig_id'].values)
 submission.shape
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )<load_from_csv>
shops = shop[['shop_id', 'city', 'type']]
kernels = pd.read_csv('/kaggle/input/moa-public-kernels/kernels.csv', index_col='id')
 kernels.head(10 )<load_from_csv>
from sklearn.preprocessing import LabelEncoder
def make_filename(idx):
return glob.glob('/kaggle/input/moa-public-kernels/' + str(idx)+ '__submission.csv')[0]
def read_predictions(idx):
temp = pd.read_csv(make_filename(idx))
return temp
predictions = [read_predictions(idx)for idx in kernels.index]
len(predictions )<load_from_csv>
shops["city"] = LabelEncoder().fit_transform(shops.city)
 shops["type"] = LabelEncoder().fit_transform(shops.type )
sub = pd.read_csv('.. /input/pytorch-new/submission.csv')
 sub = sub.drop(['sig_id'],axis = 1)
 sub.head(5)
 predictions.append(sub )<prepare_x_and_y>
def clean_name(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
columns = list(submission.columns)
 columns.remove('sig_id')
 y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape<save_to_csv>
items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: clean_name(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )
columns = list(submission.columns)
 columns.remove('sig_id')
 submission = pd.DataFrame(index = public_ids + private_ids, columns=columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
 submission.head().T<set_options>
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN 
<set_options>
group_sum = items.groupby(["type"], as_index=False ).agg({"item_id": "count"})
 group_sum
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<load_from_csv>
drop_cols = group_sum.loc[group_sum['item_id']<40,'type'].values.tolist()
def preprocessor_TabNet() :
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 train_drug = pd.read_csv(data_path + "train_drug.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
train = train.merge(targets, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 target_cols = [x for x in targets.columns if x != 'sig_id']
train_df = train
train_df.reset_index(drop=True, inplace=True)
 test.reset_index(drop = True, inplace = True)
 features_to_drop = ["sig_id", "cp_type"]
test.drop(features_to_drop, axis = 1, inplace = True)
 feature_cols = [c for c in train_df.columns if c not in targets.columns]
feature_cols = [c for c in feature_cols if c not in [ 'sig_id', 'drug_id', 'cp_type']]
return train, targets, test, submission, target_cols, feature_cols<define_search_model>
items.name2 = items.name2.apply(lambda x: "etc" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = [x for x in range(42,49)]
NFOLDS = 10
DRUG_THRESH = 18
train_df = make_cv_folds(train_df, SEEDS, NFOLDS, DRUG_THRESH )<choose_model_class>
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()
X_test = test_TabNet.values
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10 
)
 class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux)
 test_cv_preds = []
for s in SEEDS:
tabnet_params['seed'] = s
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/TabNet_seed_'\
+str(s)+'_fold_'+str(fold_id+1)
 saved_file = '/kaggle/working/model'
shutil.make_archive(saved_file,'zip',saved_path_name)
 print(f'Successfully saved model at {saved_file}.zip')
 device =('cuda' if torch.cuda.is_available() else 'cpu')
 with zipfile.ZipFile('model.zip')as z:
with z.open('model_params.json')as f:
loaded_params = json.load(f)
 with z.open('network.pt')as f:
try:
saved_state_dict = torch.load(f, map_location=device)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.bytesIO(f.read()),map_location = device)
 loaded_model = TabNetRegressor(**loaded_params)
 loaded_model._set_network()
loaded_model.network.load_state_dict(saved_state_dict)
 loaded_model.network.eval()
preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/'
oof_TabNet_all = np.load(saved_path_name + 'oof_TabNet_all.npy' )<load_from_csv>
ts = time.time()
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True)
 time.time() - ts
data_path = ".. /input/lish-moa/"
all_feat = [col for col in submission_TabNet.columns if col not in ["sig_id"]]
test_TabNet = pd.read_csv(data_path + "test_features.csv")
 sig_id = test_TabNet[test_TabNet["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission_TabNet = pd.merge(test_TabNet[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission_TabNet.fillna(0, inplace = True)
 submission_TabNet.to_csv("submission_TabNet.csv", index = None)
 submission_TabNet.head()<drop_column>
train["revenue"] = train["item_cnt_day"] * train["item_price"]
del targets_TabNet, test_TabNet, tmp, train_df
gc.collect()<set_options>
ts = time.time()
group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.columns = ["item_cnt_month"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = cols, how = "left")
 matrix["item_cnt_month"] = matrix["item_cnt_month"].fillna(0 ).clip(0,20 ).astype(np.float16)
 time.time() - ts
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed)
 def preprocessor_nn_transfer() :
variance_threshould = 0.8
ncompo_genes = 600
ncompo_cells = 50
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 train_features = train_features[train_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_features = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_features = train_features.drop('cp_type', axis=1)
 test_features = test_features.drop('cp_type', axis=1)
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train_features[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train_features[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train_features = pd.concat([train_features, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test_features[GENES])
 pca_cells_test = pca_cells.transform(test_features[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test_features = pd.concat([test_features, pca_genes_test, pca_cells_test], axis = 1)
 PCA')
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
mask =(train_features[cols_numeric].var() >= variance_threshould ).values
tmp = train_features[cols_numeric].loc[:, mask]
train_features = pd.concat([train_features[["sig_id", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
test_features = pd.concat([test_features[["sig_id", "cp_time", "cp_dose"]], test_features.loc[:,cols_numeric]], axis = 1)
 train_features = pd.get_dummies(train_features, columns = ['cp_time', 'cp_dose'])
 test_features = pd.get_dummies(test_features, columns = ['cp_time', 'cp_dose'])
 Feature selection')
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 test = test_features
Join')
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 num_targets: {}'.format(num_targets))
')
 return train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols<split>
test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
 test["shop_id"] = test.shop_id.astype(np.int8)
 test["item_id"] = test.item_id.astype(np.int16 )
train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols = preprocessor_nn_transfer()<categorify>
ts = time.time()
matrix = pd.concat([matrix, test.drop(["ID"],axis = 1)], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace = True)
 time.time() - ts
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<define_variables>
ts = time.time()
matrix = pd.merge(matrix, shops, on = ["shop_id"], how = "left")
 matrix = pd.merge(matrix, items, on = ["item_id"], how = "left")
 matrix["city"] = matrix["city"].astype(np.int8)
 matrix["type"] = matrix["type"].astype(np.int8)
 matrix["item_category_id"] = matrix["item_category_id"].astype(np.int8)
 matrix["name2"] = matrix["name2"].astype(np.int8)
 matrix["name3"] = matrix["name3"].astype(np.int16)
 time.time() - ts
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 model = Model(num_features, num_all_targets)
 def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 display(train.head())
 def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = train
test_ = test
kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transfer-learning-oof/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return 0.02, predictions<categorify>
def lag_feature(df,lags, cols):
for col in cols:
print(col)
 tmp = df[["date_block_num", "shop_id","item_id",col ]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ["date_block_num", "shop_id", "item_id", col + "_lag_"+str(i)]
shifted.date_block_num = shifted.date_block_num + i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
str(timedelta(seconds=time_diff))
data_dir = '.. /input/lish-moa/'
train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_scored.head()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1])
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 sub_nn_transfer = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub_nn_transfer.to_csv('submission_nn_transfer.csv', index=False )<load_pretrained>
ts = time.time()
group = matrix.groupby(["date_block_num"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1], ["date_avg_item_cnt"])
 matrix.drop(["date_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
saved_path_name = '.. /input/nn-transfer-learning-oof/'
oof_nn_transfer = np.load(saved_path_name + 'oof_nn_transfer_all.npy')
 oof_nn_transfer.shape<drop_column>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix.date_item_avg_item_cnt = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ['date_item_avg_item_cnt'])
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
del sample_submission, test, train, train_targets_scored, valid_results
gc.collect()<load_from_csv>
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_avg_item_cnt"])
 matrix.drop(["date_shop_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
def readin_data_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets_drug = train_targets_scored.merge(train_drug, on='sig_id')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
targets_drug = targets_drug.iloc[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/ttestpcarfelogisticregression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,labels_train,test_features,targets_drug,predictors,cs,gs 
<compute_test_metric>
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id","item_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_item_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id","item_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_item_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_item_avg_item_cnt"])
 matrix.drop(["date_shop_item_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def resnet_preprocessor(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test 
<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id'], how='left')
 matrix.date_shop_subtype_avg_item_cnt = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_shop_subtype_avg_item_cnt'])
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
def resnet_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.25),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model<sort_values>
ts = time.time()
group = matrix.groupby(['date_block_num', 'city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_city_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', "city"], how='left')
 matrix.date_city_avg_item_cnt = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_city_avg_item_cnt'])
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
def resnet_make_cv_folds(targets_drug,target_cols, SEEDS, NFOLDS, DRUG_THRESH):
vc = targets_drug.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.loc[targets_drug.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 targets_drug[kfold_col] = targets_drug.drug_id.map(dct1)
 targets_drug.loc[targets_drug[kfold_col].isna() , kfold_col] = targets_drug.loc[targets_drug[kfold_col].isna() , 'sig_id'].map(dct2)
 targets_drug[kfold_col] = targets_drug[kfold_col].astype('int8')
 return targets_drug 
<predict_on_test>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city'], how='left')
 matrix.date_item_city_avg_item_cnt = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_item_city_avg_item_cnt'])
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
def predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs):
n_labels = labels_train.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
n_seeds = len(SEEDS)
 n_folds = NFOLDS
y_pred = np.zeros(( n_test,n_labels))
oof = np.zeros(( n_train,n_labels))
train_all,data_test = resnet_preprocessor(train_features.values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 for seed in SEEDS:
kfold_col = 'kfold_{}'.format(seed)
 kf = targets_drug1[kfold_col]
for fold in range(NFOLDS):
train = targets_drug1[kfold_col] != fold
test = targets_drug1[kfold_col] == fold
X_train, X_test = train_all[train],train_all[test]
X_train_2 = train_features[train][predictors].values
X_test_2 = train_features[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = keras.models.load_model('/kaggle/input/resnet-001843/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold), custom_objects={'logloss':logloss})
 y_val = model.predict([X_test,X_test_2])
 oof[test] += y_val/ n_seeds
y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 return oof,y_pred 
<find_best_model_class>
ts = time.time()
group = train.groupby(["item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["item_id"], how = "left")
 matrix["item_avg_item_price"] = matrix.item_avg_item_price.astype(np.float16)
 group = train.groupby(["date_block_num","item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["date_item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["date_block_num","item_id"], how = "left")
 matrix["date_item_avg_item_price"] = matrix.date_item_avg_item_price.astype(np.float16)
 lags = [1, 2, 3]
matrix = lag_feature(matrix, lags, ["date_item_avg_item_price"])
 for i in lags:
matrix["delta_price_lag_" + str(i)] =(matrix["date_item_avg_item_price_lag_" + str(i)]- matrix["item_avg_item_price"])/ matrix["item_avg_item_price"]
def select_trends(row):
for i in lags:
if row["delta_price_lag_" + str(i)]:
return row["delta_price_lag_" + str(i)]
return 0
matrix["delta_price_lag"] = matrix.apply(select_trends, axis = 1)
 matrix["delta_price_lag"] = matrix.delta_price_lag.astype(np.float16)
 matrix["delta_price_lag"].fillna(0 ,inplace = True)
 features_to_drop = ["item_avg_item_price", "date_item_avg_item_price"]
for i in lags:
features_to_drop.append("date_item_avg_item_price_lag_" + str(i))
features_to_drop.append("delta_price_lag_" + str(i))
matrix.drop(features_to_drop, axis = 1, inplace = True)
 time.time() - ts
train_features,labels_train,test_features,targets_drug,predictors,cs,gs = readin_data_resnet()
p_min = 0.0005
p_max = 0.9995
DRUG_THRESH = 18
n_seeds = 7
np.random.seed(42)
 SEEDS = np.random.randint(0,100,size=n_seeds)
 NFOLDS = 10
target_cols = targets_drug.drop(['sig_id','drug_id'], axis=1 ).columns.values.tolist()
targets_drug_copy = targets_drug.copy()
targets_drug1 = resnet_make_cv_folds(targets_drug_copy,target_cols, SEEDS, NFOLDS, DRUG_THRESH)
 oof_rs, test_pred_rs = predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs )<save_to_csv>
ts = time.time()
group = train.groupby(["date_block_num","shop_id"] ).agg({"revenue": ["sum"] })
 group.columns = ["date_shop_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group , on = ["date_block_num", "shop_id"], how = "left")
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(["shop_id"] ).agg({ "date_block_num":["mean"] })
 group.columns = ["shop_avg_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["shop_id"], how = "left")
 matrix["shop_avg_revenue"] = matrix.shop_avg_revenue.astype(np.float32)
 matrix["delta_revenue"] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix["delta_revenue"] = matrix["delta_revenue"].astype(np.float32)
 matrix = lag_feature(matrix, [1], ["delta_revenue"])
 matrix["delta_revenue_lag_1"] = matrix["delta_revenue_lag_1"].astype(np.float32)
 matrix.drop(["date_shop_revenue", "shop_avg_revenue", "delta_revenue"] ,axis = 1, inplace = True)
 time.time() - ts
sub_rs = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_rs.iloc[:,1:] = np.clip(test_pred_rs,p_min,p_max)
 sub_rs.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_rs.to_csv('submission_rs.csv', index=False )<create_dataframe>
matrix["month"] = matrix["date_block_num"] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix["days"] = matrix["month"].map(days ).astype(np.int8 )
%who DataFrame<drop_column>
ts = time.time()
matrix["item_shop_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id","shop_id"])["date_block_num"].transform('min')
 matrix["item_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id"])["date_block_num"].transform('min')
 time.time() - ts
del targets_drug, targets_drug1, targets_drug_copy, train_features
gc.collect()<load_from_csv>
matrix['Weekends'] = matrix['date_block_num'].map(n_weekends ).fillna(10)
 matrix['Holidays'] = matrix['date_block_num'].map(n_holidays ).fillna(1 )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
<normalization>
ts = time.time()
data = matrix[matrix["date_block_num"] > 3]
time.time() - ts
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def remove_ctl(no_ctl,train,test,targets):
if no_ctl:
print('not_ctl')
 train = train[train['cp_type']!='ctl_vehicle']
test = test[test['cp_type']!='ctl_vehicle']
targets = targets.iloc[train.index]
train.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 targets.reset_index(drop=True, inplace=True)
 return train,test,targets
def remove_small_variance(variance_threshould,train,test):
cols_numeric = [feat for feat in list(train.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp_train = train[cols_numeric].loc[:, mask]
tmp_test = test[cols_numeric].loc[:, mask]
train = pd.concat([train[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_train], axis=1)
 test = pd.concat([test[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_test], axis=1)
 return train,test
def pca_process(ncompo_genes,ncompo_cells,train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
pca_genes = PCA(n_components=ncompo_genes, random_state=base_seed ).fit(train[GENES])
 pca_cells = PCA(n_components=ncompo_cells, random_state=base_seed ).fit(train[CELLS])
 train_pca_gene = pca_genes.transform(train[GENES])
 test_pca_gene = pca_genes.transform(test[GENES])
 train_pca_cell = pca_cells.transform(train[CELLS])
 test_pca_cell = pca_cells.transform(test[CELLS])
 train_pca_gene = pd.DataFrame(train_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 test_pca_gene = pd.DataFrame(test_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 train_pca_cell = pd.DataFrame(train_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 test_pca_cell = pd.DataFrame(test_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 train_df = pd.concat([train, train_pca_gene, train_pca_cell], axis=1)
 test_df = pd.concat([test, test_pca_gene, test_pca_cell], axis=1)
 return train_df,test_df
def scale_data(scale, train_df,test_df):
if scale == 'minmax':
print('minmax')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 elif scale == 'rankgauss':
print('rankgauss')
 cols_numeric = [feat for feat in list(train_df.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose'] and "pca_" not in feat]
scaler = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train_df[cols_numeric] = scaler.fit_transform(train_df[cols_numeric])
 test_df[cols_numeric] = scaler.transform(test_df[cols_numeric])
 else:
pass
return train_df,test_df
def add_features(train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
for stats in tqdm.tqdm(['sum', 'mean', 'std', 'kurt', 'skew']):
train['g_'+stats] = getattr(train[GENES], stats )(axis=1)
 train['c_'+stats] = getattr(train[CELLS], stats )(axis=1)
 train['gc_'+stats] = getattr(train[GENES+CELLS], stats )(axis=1)
 test['g_'+stats] = getattr(test[GENES], stats )(axis=1)
 test['c_'+stats] = getattr(test[CELLS], stats )(axis=1)
 test['gc_'+stats] = getattr(test[GENES+CELLS], stats )(axis=1)
 return train,test
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def data_processing(train,test, targets):
train,test,targets = remove_ctl(no_ctl,train,test,targets)
 print("remove control: ", train.shape,test.shape)
 train,test = remove_small_variance(variance_threshould,train,test)
 print("remove small_var: ", train.shape,test.shape)
 train_df,test_df = pca_process(ncompo_genes,ncompo_cells,train,test)
 print("do PCA: ", train_df.shape,test_df.shape)
 train_df,test_df = scale_data(scale, train_df,test_df)
 print("rankGauss: ", train_df.shape,test_df.shape)
 train_df,test_df = add_features(train_df,test_df)
 print("add stats feature:", train_df.shape,test_df.shape)
 return train_df,test_df,targets<groupby>
from xgboost import XGBRegressor
import pickle
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train<init_hyperparams>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.8
decompo = 'PCA'
ncompo_genes = 50
ncompo_cells = 8
encoding = 'dummy'
base_seed = 42
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 20
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=1024
SEEDS = 7
SEED = [0, 1, 2, 3, 4, 5, 6]
DRUG_THRESH = 18<merge>
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 20 )
seed_everything(base_seed)
 train, test, targets = data_processing(train_features,test_features, train_targets_scored)
 target_cols = train_targets_scored.drop('sig_id', axis=1 ).columns.values.tolist()
print(len(target_cols))
train_comb = train.merge(targets, on='sig_id')
 train_comb.drop('cp_type', axis=1, inplace=True)
 print("combine train and targets to prepare for kfolds ", train_comb.shape)
 feature_cols = [c for c in process_data(train_comb ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
print(len(feature_cols))
gen_cols = [c for c in feature_cols if 'g-' == c[:2]]
cell_cols = [c for c in feature_cols if 'c-' == c[:2]]
pca_gen_cols = [c for c in feature_cols if 'pca_g' == c[:5]]
pca_cell_cols = [c for c in feature_cols if 'pca_c' == c[:5]]
ohe_cols = [c for c in feature_cols if c not in gen_cols+cell_cols+pca_gen_cols+pca_cell_cols]
print(len(gen_cols), len(cell_cols), len(pca_gen_cols), len(pca_cell_cols),len(ohe_cols))
num_features=len(feature_cols)
 num_targets=len(target_cols)
 folds = train_comb.copy()
folds = folds.merge(train_drug, on='sig_id')
 folds = make_cv_folds(folds, SEEDS, NFOLDS, DRUG_THRESH )<choose_model_class>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))