Kaggle competition: Google Analytics Customer Revenue Prediction
<SOS> metric: RMSE Kaggle data source: google-analytics-customer-revenue-prediction<define_variables>
%matplotlib inline
pd.options.display.max_columns = 999
plt.rcParams['figure.figsize'] =(14, 7)
 font = {'family' : 'verdana',
'weight' : 'bold',
'size' : 14}
plt.rc('font', **font)
 warnings.simplefilter("ignore")
 gc.enable()
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
 train.shape, test.shape
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024 
<prepare_x_and_y>
train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'} )
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(np.int64 )
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
train = train.merge(pd.concat([train_store_1, train_store_2], sort=False), how="left", on="visitId")
 test = test.merge(pd.concat([test_store_1, test_store_2], sort=False), how="left", on="visitId")
 for df in [train, test]:
df.drop("Client Id", 1, inplace=True )
SEED = [0, 1, 2, 3 ,4, 5]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<merge>
for df in [train, test]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
<save_to_csv>
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
del df
gc.collect()
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<load_from_csv>
target_sums = train.groupby("fullVisitorId")["totals.transactionRevenue"].sum().reset_index()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<categorify>
ids_train = set(train.fullVisitorId.unique())
 ids_test = set(test.fullVisitorId.unique())
 print("Unique visitor ids in train:", len(ids_train))
print("Unique visitor ids in test:", len(ids_test))
print("Common visitors in train and test:", len(ids_train & ids_test))
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72:2})
 del df['sig_id']
return df
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<import_modules>
problem = train[train.sessionId.map(train.sessionId.value_counts() == 2)].sort_values(["sessionId", 'visitStartTime'])
 problem.head(10 )
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau,ModelCheckpoint,EarlyStopping,LearningRateScheduler
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
from tqdm.notebook import tqdm
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LeakyReLU,PReLU,ELU
from keras.layers import Dropout<choose_model_class>
( train.visitStartTime == train.visitId ).value_counts()
def create_model(num_columns):
classifier = Sequential()
classifier.add(keras.layers.BatchNormalization())
 classifier.add(Dense(units = 900, kernel_initializer = 'he_uniform',activation='relu',input_dim = num_columns))
classifier.add(keras.layers.BatchNormalization())
 classifier.add(keras.layers.Dropout (.3))
classifier.add(Dense(units = 900, kernel_initializer = 'he_uniform',activation='relu'))
classifier.add(keras.layers.BatchNormalization())
 classifier.add(keras.layers.Dropout (.2))
classifier.add(Dense(units = 206, kernel_initializer = 'glorot_uniform', activation = 'sigmoid'))
optimizer = tfa.optimizers.Lookahead('adam',sync_period=10)
 classifier.compile(optimizer=optimizer,loss='binary_crossentropy', metrics=['accuracy'])
 return classifier<define_variables>
train.loc[pd.to_datetime(train.visitStartTime, unit='s')== "2017-04-25 18:49:35"].head(8 )
top_feats = [ 0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 14, 15,
16, 18, 19, 20, 21, 23, 24, 25, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 44, 45, 46,
48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76,
78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 92,
93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 105, 106, 107,
108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120,
121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 132, 133, 134,
135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
149, 150, 151, 152, 153, 154, 155, 157, 159, 160, 161, 163, 164,
165, 166, 167, 168, 169, 170, 172, 173, 175, 176, 177, 178, 180,
181, 182, 183, 184, 186, 187, 188, 189, 190, 191, 192, 193, 195,
197, 198, 199, 202, 203, 205, 206, 208, 209, 210, 211, 212, 213,
214, 215, 218, 219, 220, 221, 222, 224, 225, 227, 228, 229, 230,
231, 232, 233, 234, 236, 238, 239, 240, 241, 242, 243, 244, 245,
246, 248, 249, 250, 251, 253, 254, 255, 256, 257, 258, 259, 260,
261, 263, 265, 266, 268, 270, 271, 272, 273, 275, 276, 277, 279,
282, 283, 286, 287, 288, 289, 290, 294, 295, 296, 297, 299, 300,
301, 302, 303, 304, 305, 306, 308, 309, 310, 311, 312, 313, 315,
316, 317, 320, 321, 322, 324, 325, 326, 327, 328, 329, 330, 331,
332, 333, 334, 335, 338, 339, 340, 341, 343, 344, 345, 346, 347,
349, 350, 351, 352, 353, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 368, 369, 370, 371, 372, 374, 375, 376, 377,
378, 379, 380, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391,
392, 393, 394, 395, 397, 398, 399, 400, 401, 403, 405, 406, 407,
408, 410, 411, 412, 413, 414, 415, 417, 418, 419, 420, 421, 422,
423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435,
436, 437, 438, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450,
452, 453, 454, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465,
466, 468, 469, 471, 472, 473, 474, 475, 476, 477, 478, 479, 482,
483, 485, 486, 487, 488, 489, 491, 492, 494, 495, 496, 500, 501,
502, 503, 505, 506, 507, 509, 510, 511, 512, 513, 514, 516, 517,
518, 519, 521, 523, 525, 526, 527, 528, 529, 530, 531, 532, 533,
534, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547,
549, 550, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563,
564, 565, 566, 567, 569, 570, 571, 572, 573, 574, 575, 577, 580,
581, 582, 583, 586, 587, 590, 591, 592, 593, 595, 596, 597, 598,
599, 600, 601, 602, 603, 605, 607, 608, 609, 611, 612, 613, 614,
615, 616, 617, 619, 622, 623, 625, 627, 630, 631, 632, 633, 634,
635, 637, 638, 639, 642, 643, 644, 645, 646, 647, 649, 650, 651,
652, 654, 655, 658, 659, 660, 661, 662, 663, 664, 666, 667, 668,
669, 670, 672, 674, 675, 676, 677, 678, 680, 681, 682, 684, 685,
686, 687, 688, 689, 691, 692, 694, 695, 696, 697, 699, 700, 701,
702, 703, 704, 705, 707, 708, 709, 711, 712, 713, 714, 715, 716,
717, 723, 725, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738,
739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751,
752, 753, 754, 755, 756, 758, 759, 760, 761, 762, 763, 764, 765,
766, 767, 769, 770, 771, 772, 774, 775, 780, 781, 782, 783, 784,
785, 787, 788, 790, 793, 795, 797, 799, 800, 801, 805, 808, 809,
811, 812, 813, 816, 819, 820, 821, 822, 823, 825, 826, 827, 829,
831, 832, 833, 834, 835, 837, 838, 839, 840, 841, 842, 844, 845,
846, 847, 848, 850, 851, 852, 854, 855, 856, 858, 860, 861, 862,
864, 867, 868, 870, 871, 873, 874]
print(len(top_feats))<split>
print("Train: ", np.bincount(train.visitId.value_counts()))
N_STARTS = 9
res = train_targets.copy()
ss.loc[:, train_targets.columns] = 0
res.loc[:, train_targets.columns] = 0
historys = dict()
tf.random.set_seed(43)
 for seed in range(N_STARTS):
for n,(tr, te)in enumerate(KFold(n_splits=5, random_state=(seed+1)**2, shuffle=True ).split(train_targets, train_targets)) :
print(f"======{train_targets.values[tr].shape}========{train_targets.values[te].shape}=====")
 print(f'Seed: {seed} => Fold: {n}')
 model = create_model(len(top_feats))
checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-5, patience=3, verbose=1, mode='min')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'min')
 early = EarlyStopping(monitor="val_loss", mode="min", restore_best_weights=True, patience= 5, verbose = 1)
 history = model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=100, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt, early], verbose=2 
)
 historys[f'history_{seed+1}'] = history
print("Model History Saved.")
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss.loc[:, train_targets.columns] += test_predict
res.loc[te, train_targets.columns] += val_predict
ss.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res.loc[:, train_targets.columns] /= N_STARTS<compute_test_metric>
print("test: ", np.bincount(test.visitId.value_counts()))
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(train_targets.loc[:, _target], res.loc[:, _target]))
print(f'OOF Metric: {np.mean(metrics)}' )<compute_train_metric>
train.visitStartTime = pd.to_datetime(train.visitStartTime, unit='s')
 test.visitStartTime = pd.to_datetime(test.visitStartTime, unit='s')
 train["date"] = train.visitStartTime
test["date"] = test.visitStartTime
metrics = []
res.loc[train['cp_type']==1, train_targets.columns] = 0
for _target in train_targets.columns:
metrics.append(log_loss(train_targets.loc[:, _target], res.loc[:, _target]))
print(f'OOF Metric with postprocessing: {np.mean(metrics)}' )<feature_engineering>
train.set_index("visitStartTime", inplace=True)
 test.set_index("visitStartTime", inplace=True)
 train.sort_index(inplace=True)
 test.sort_index(inplace=True )
ss.loc[test['cp_type']==1, train_targets.columns] = 0<save_to_csv>
def clearRare(columnname, limit = 1000):
vc = test[columnname].value_counts()
common = vc > limit
common = set(common.index[common].values)
 print("Set", sum(vc <= limit), columnname, "categories to 'other';", end=" ")
 train.loc[train[columnname].map(lambda x: x not in common), columnname] = 'other'
test.loc[test[columnname].map(lambda x: x not in common), columnname] = 'other'
print("now there are", train[columnname].nunique() , "categories in train" )
ss.to_csv('submission.csv', index=False )<import_modules>
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True )
import numpy as np
import pandas as pd<load_from_csv>
clearRare("device.browser")
 clearRare("device.operatingSystem")
 clearRare("geoNetwork.country")
 clearRare("geoNetwork.city")
 clearRare("geoNetwork.metro")
 clearRare("geoNetwork.networkDomain")
 clearRare("geoNetwork.region")
 clearRare("geoNetwork.subContinent")
 clearRare("trafficSource.adContent")
 clearRare("trafficSource.campaign")
 clearRare("trafficSource.keyword")
 clearRare("trafficSource.medium")
 clearRare("trafficSource.referralPath")
 clearRare("trafficSource.source" )
df_under = pd.read_csv('/kaggle/input/best-subs-moa/submission_Under.csv')
 df_medium = pd.read_csv('/kaggle/input/best-subs-moa/submission_medium.csv')
 df_over = pd.read_csv('/kaggle/input/best-subs-moa/submission_over_over.csv' )<sort_values>
for df in [train, test]:
df["Avg.Session Duration"][df["Avg.Session Duration"] == 0] = "00:00:00"
df["Avg.Session Duration"] = df["Avg.Session Duration"].str.split(':' ).apply(lambda x: int(x[0])* 60 + int(x[1]))
df["Bounce Rate"] = df["Bounce Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float )
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<concatenate>
for df in [train, test]:
df["id_incoherence"] = pd.to_datetime(df.visitId, unit='s')!= df.date
df["visitId_dublicates"] = df.visitId.map(df.visitId.value_counts())
 df["session_dublicates"] = df.sessionId.map(df.sessionId.value_counts() )
df_over = order_sub(df_over)
 df_medium = order_sub(df_medium)
 df_under = order_sub(df_under )<prepare_output>
for df in [train, test]:
df['weekday'] = df['date'].dt.dayofweek.astype(object)
 df['time'] = df['date'].dt.second + df['date'].dt.minute*60 + df['date'].dt.hour*3600
df['day'] = df['date'].dt.date
BLEND =df_over.copy()
BLEND.iloc[:,1:] = df_over.iloc[:,1:]*0.80 +df_medium.iloc[:,1:]*0.15 +df_under.iloc[:,1:]*0.05<define_variables>
df = pd.concat([train, test])
 df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)
 df['prev_session'] =(df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)).astype(np.int64)// 1e9 // 60 // 60
df['next_session'] =(df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)).astype(np.int64)// 1e9 // 60 // 60
df.sort_index(inplace=True)
 train = df[:len(train)]
test = df[len(train):]
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL)
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<compute_test_metric>
for df in [train, test]:
df['source.country'] = df['trafficSource.source'] + '_' + df['geoNetwork.country']
df['campaign.medium'] = df['trafficSource.campaign'] + '_' + df['trafficSource.medium']
df['browser.category'] = df['device.browser'] + '_' + df['device.deviceCategory']
df['browser.os'] = df['device.browser'] + '_' + df['device.operatingSystem']
def get_mean_of_subs(sub1,sub2,sub3):
s=1
for i,j,k in zip(sub1.iloc[:,1:].columns, sub2.iloc[:,1:].columns, sub3.iloc[:,1:].columns):
print('
print('sub1' + ' ' + ' Columns :' + str(i)+ ' MEAN :',sub1[i].mean())
 print('sub2' + ' ' + ' Columns :' + str(j)+ ' MEAN :',sub2[j].mean())
 print('sub3' + ' ' + ' Columns :' + str(k)+ ' MEAN :',sub3[k].mean())
 print('
')
 s+=1<compute_test_metric>
for df in [train, test]:
df['device_deviceCategory_channelGrouping'] = df['device.deviceCategory'] + "_" + df['channelGrouping']
df['channelGrouping_browser'] = df['device.browser'] + "_" + df['channelGrouping']
df['channelGrouping_OS'] = df['device.operatingSystem'] + "_" + df['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
df[i + "_" + j] = df[i] + "_" + df[j]
df['content.source'] = df['trafficSource.adContent'].astype(str)+ "_" + df['source.country']
df['medium.source'] = df['trafficSource.medium'] + "_" + df['source.country']
get_mean_of_subs(df_over,df_medium,df_under )<define_variables>
for feature in ["totals.hits", "totals.pageviews"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].mean()
train["usermean_" + feature] = train.fullVisitorId.map(info)
 test["usermean_" + feature] = test.fullVisitorId.map(info)
 for feature in ["visitNumber"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].max()
train["usermax_" + feature] = train.fullVisitorId.map(info)
 test["usermax_" + feature] = test.fullVisitorId.map(info )
List_coefs = [[0.82,0.13,0.05],[0.9,0.05,0.05],[0.7,0.25,0.05],[0.8,0.1,0.1],[0.35,0.35,0.3],[0.35,0.35,0.3],[0.4,0.3,0.3],
[0.4,0.3,0.3],[0.4,0.3,0.3],[0.8,0.15,0.05],[0.9,0.05,0.05],[0.35,0.35,0.3],[0.9,0.05,0.05],[0.72,0.25,0.03],
[0.9,0.05,0.05],[0.72,0.25,0.03],[0.9,0.09,0.01],[0.9,0.09,0.01],[0.9,0.01,0.09],[0.9,0.01,0.09],[0.95,0.03,0.02],
[0.95,0.03,0.02],[0.9,0.1,0.00],[0.9,0.09,0.01],[0.9,0.1,0.00],[0.9,0.1,0.00],[0.9,0.1,0.00],[0.9,0.1,0.00],
[0.9,0.1,0.00],[0.8,0.05,0.15],[0.95,0.03,0.02],[0.95,0.03,0.02],[0.95,0.01,0.04]]
len(List_coefs )<feature_engineering>
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime',
'month', 'day', 'help']
cat_cols = [f for f in train.columns if(train[f].dtype == 'object' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]
BLEND =df_over.copy()
for i,coefs in enumerate(List_coefs):
BLEND[cols_with_diff_coefs[i]] = df_over[cols_with_diff_coefs[i]]* List_coefs[i][0] +df_medium[cols_with_diff_coefs[i]]* List_coefs[i][1] +df_under[cols_with_diff_coefs[i]]* List_coefs[i][2]
BLEND.iloc[:,34:] = df_over.iloc[:,34:]*0.82 +df_medium.iloc[:,34:]*0.13 +df_under.iloc[:,34:]*0.05<define_variables>
train[cat_cols].nunique()
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL)
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0 
<set_options>
for col in cat_cols:
lbl = LabelEncoder()
lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))
 train[col] = lbl.transform(list(train[col].values.astype('str')))
 test[col] = lbl.transform(list(test[col].values.astype('str')) )
warnings.filterwarnings("ignore" )<import_modules>
for col in real_cols:
train[col] = train[col].astype(float)
 test[col] = test[col].astype(float )
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler
from tensorflow.keras.losses import BinaryCrossentropy
import tensorflow_addons as tfa
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
from sklearn import preprocessing
from tqdm.notebook import tqdm
import math<load_from_csv>
for to_del in ["date", "sessionId", "visitId", "day"]:
del train[to_del]
del test[to_del]
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 data = train_features.append(test_features)
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<categorify>
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime', "month", "help"]
cat_cols = [f for f in train.columns if(train[f].dtype == 'int64' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]
def preprocess(df):
df = df.copy()
df.drop(['cp_type'], axis=1, inplace=True)
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 df = pd.get_dummies(df, columns=['cp_time','cp_dose'])
 del df['sig_id']
return df
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<normalization>
def score(data, y):
validation_res = pd.DataFrame(
 {"fullVisitorId": data["fullVisitorId"].values,
"transactionRevenue": data["totals.transactionRevenue"].values,
"predictedRevenue": np.expm1(y)})
 validation_res = validation_res.groupby("fullVisitorId")["transactionRevenue", "predictedRevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionRevenue"].values),
np.log1p(validation_res["predictedRevenue"].values)) )
scaler = preprocessing.MinMaxScaler()
scaler.fit(train.append(test))
train_trans = scaler.transform(train)
 test_trans = scaler.transform(test)
 train = pd.DataFrame(train_trans, columns=train.columns)
 test = pd.DataFrame(test_trans, columns=test.columns )<compute_test_metric>
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['fullVisitorId'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 50, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["totals.transactionRevenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["totals.transactionRevenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score
somthing_rate = 1e-15
P_MIN = somthing_rate
P_MAX = 1 - P_MIN
def loss_fn(yt, yp):
yp = np.clip(yp, P_MIN, P_MAX)
 return log_loss(yt, yp, labels=[0,1])
<choose_model_class>
Kfolder = KFoldValidation(train )
def create_model(num_columns, actv='relu'):
model = tf.keras.Sequential([tf.keras.layers.Input(num_columns)])
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation=actv)))
 if actv == 'elu':
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.AlphaDropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, kernel_initializer='lecun_normal', activation='selu')))
 else:
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation=actv)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid")))
 model.compile(optimizer=tfa.optimizers.AdamW(lr = 1e-3, weight_decay = 1e-5, clipvalue = 756),
loss=BinaryCrossentropy(label_smoothing=somthing_rate), 
)
 return model<compute_test_metric>
lgbmodel = lgb.LGBMRegressor(n_estimators=1000, objective="regression", metric="rmse", num_leaves=31, min_child_samples=100,
learning_rate=0.03, bagging_fraction=0.7, feature_fraction=0.5, bagging_frequency=5,
bagging_seed=2019, subsample=.9, colsample_bytree=.9, use_best_model=True )
def metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(loss_fn(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float)))
 return np.mean(metrics )<feature_engineering>
Kfolder.validate(train, test, real_cols + cat_cols, lgbmodel, "lgbpred", prepare_stacking=True )
N_STARTS = 14
S_STARTS = int(N_STARTS/2)
 res_relu = train_targets.copy()
res_elu = train_targets.copy()
res_relu.loc[:, train_targets.columns] = 0
res_elu.loc[:, train_targets.columns] = 0
ss_relu = ss.copy()
ss_elu = ss.copy()
ss_relu.loc[:, train_targets.columns] = 0
ss_elu.loc[:, train_targets.columns] = 0
ss_dict = {}
historys = dict()
tf.random.set_seed(42)
 for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=7, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f"======{train_targets.values[tr].shape}========{train_targets.values[te].shape}=====")
 if seed < S_STARTS:
print(f'Seed: {seed} => Fold: {n} ==>(RELU MODEL)')
 model = create_model(len(top_feats), actv='relu')
 else:
print(f'Seed: {seed} => Fold: {n} ==>(ELU MODEL)')
 model = create_model(len(top_feats), actv='elu')
 checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.2, min_lr=1e-6, patience=4, verbose=1, mode='auto')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'auto')
 early = EarlyStopping(monitor="val_loss", mode="min", restore_best_weights=True, patience= 10, verbose = 1)
 history = model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=60, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt, early], verbose=2 
)
 historys[f'history_{seed+1}'] = history
print("Model History Saved.")
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 if seed < S_STARTS:
ss_relu.loc[:, train_targets.columns] += test_predict
res_relu.loc[te, train_targets.columns] += val_predict
else:
ss_elu.loc[:, train_targets.columns] += test_predict
res_elu.loc[te, train_targets.columns] += val_predict
print(f'OOF Metric For SEED {seed} => FOLD {n} : {metric(train_targets.loc[te, train_targets.columns], pd.DataFrame(val_predict, columns=train_targets.columns)) }')
 print('+-' * 10)
 ss_relu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_relu.loc[:, train_targets.columns] /= S_STARTS
ss_elu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_elu.loc[:, train_targets.columns] /= S_STARTS<compute_test_metric>
def create_user_df(df):
agg_data = df[real_cols + cat_cols + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
pred_list = df[['fullVisitorId', 'lgbpred']].groupby('fullVisitorId' ).apply(lambda visitor_df: list(visitor_df.lgbpred)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 all_predictions = pd.DataFrame(list(pred_list.values), index=agg_data.index)
 feats = all_predictions.columns
all_predictions['t_mean'] = all_predictions.mean(axis=1)
 all_predictions['t_median'] = all_predictions.median(axis=1)
 all_predictions['t_sum_log'] = all_predictions.sum(axis=1)
 all_predictions['t_sum_act'] = all_predictions.fillna(0 ).sum(axis=1)
 all_predictions['t_nb_sess'] = all_predictions.isnull().sum(axis=1)
 full_data = pd.concat([agg_data, all_predictions], axis=1 ).astype(float)
 full_data['fullVisitorId'] = full_data.index
del agg_data, all_predictions
gc.collect()
return full_data
print(f'OOF Metric(relu): {metric(train_targets, res_relu)}')
 print(f'OOF Metric(elu): {metric(train_targets, res_elu)}' )<save_to_csv>
user_train = create_user_df(train)
 user_test = create_user_df(test )
ss_relu.to_csv('submission_relu.csv', index=False)
 ss_elu.to_csv('submission_elu.csv', index=False )<import_modules>
features = list(user_train.columns)[:-1]
user_train["totals.transactionRevenue"] = train[['fullVisitorId', 'totals.transactionRevenue']].groupby('fullVisitorId' ).sum()
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import log_loss
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow_addons as tfa
from tqdm.notebook import tqdm
import tensorflow as tf
import pandas as pd
import numpy as np
import random
import time<load_from_csv>
for f in features:
if f not in user_test.columns:
user_test[f] = np.nan
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_targets = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 del test_features['sig_id'], train_features["sig_id"], train_targets['sig_id']<categorify>
Kfolder = KFoldValidation(user_train )
for feature in ['cp_type', 'cp_dose']:
trans = LabelEncoder()
trans.fit(list(train_features[feature].astype(str ).values)+ list(test_features[feature].astype(str ).values))
train_features[feature] = trans.transform(list(train_features[feature].astype(str ).values))
test_features[feature] = trans.transform(list(test_features[feature].astype(str ).values))
train = train_features.copy()<choose_model_class>
lgbmodel = lgb.LGBMRegressor(n_estimators=1000, objective="regression", metric="rmse", num_leaves=31, min_child_samples=100,
learning_rate=0.03, bagging_fraction=0.7, feature_fraction=0.5, bagging_frequency=5,
bagging_seed=2019, subsample=.9, colsample_bytree=.9,
use_best_model=True )
def create_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1400, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(840, activation="sigmoid")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(588, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(470, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=5),
loss='binary_crossentropy', metrics=["accuracy"] 
)
 return model<choose_model_class>
Kfolder.validate(user_train, user_test, features, lgbmodel, name="lgbfinal", prepare_stacking=True )
N_STARTS = 3
tf.random.set_seed(43)
 res = train_targets.copy()
submission.loc[:, train_targets.columns] = 0
res.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(KFold(n_splits=3, random_state=seed, shuffle=True ).split(train_targets)) :
print(f'Fold {n}')
 model = create_model(len(train.columns))
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.15, patience=3, verbose=1, epsilon=1e-4, mode='min')
 early_stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, mode= 'min')
 model.fit(train.values[tr],
train_targets.values[tr],
validation_data=(train.values[te], train_targets.values[te]),
epochs=45, batch_size=128,
callbacks=[reduce_lr_loss, early_stop], verbose=2 
)
 test_predict = model.predict(test_features.values)
 val_predict = model.predict(train.values[te])
 submission.loc[:, train_targets.columns] += test_predict
res.loc[te, train_targets.columns] += val_predict
print('')
 submission.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res.loc[:, train_targets.columns] /= N_STARTS<compute_test_metric>
xgbmodel = xgb.XGBRegressor(max_depth=22, learning_rate=0.02, n_estimators=1000,
objective='reg:linear', gamma=1.45, seed=2019, silent=False,
subsample=0.67, colsample_bytree=0.054, colsample_bylevel=0.50 )
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(train_targets.loc[:, _target], res.loc[:, _target]))
print(f'OOF Metric: {np.mean(metrics)}' )<define_variables>
Kfolder.validate(user_train, user_test, features, xgbmodel, name="xgbfinal", prepare_stacking=True )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL)
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = submission.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<import_modules>
catmodel = cat.CatBoostRegressor(iterations=500, learning_rate=0.2, depth=5, random_seed=2019 )
import pandas as pd
import numpy as np
import tensorflow as tf
import tokenizers
import transformers
from tqdm import tqdm
from sklearn import metrics
from sklearn.model_selection import KFold
import re<choose_model_class>
Kfolder.validate(user_train, user_test, features, catmodel, name="catfinal", prepare_stacking=True,
fit_params={"use_best_model": True, "verbose": 100} )
MAX_LEN = 64
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 8
LEARNING_RATE = 3e-5
EPOCHS = 3
TRAINING_FILE = ".. /input/nlp-getting-started/train.csv"
TEST_FILE = ".. /input/nlp-getting-started/test.csv"
ROBERTA_PATH = ".. /input/tf-roberta"
roberta_tokenizer = transformers.RobertaTokenizer.from_pretrained('roberta-base', lower_case=True )<load_from_csv>
user_train['PredictedLogRevenue'] = 0.4 * user_train["lgbfinal"] + \
0.2 * user_train["xgbfinal"] + \
0.4 * user_train["catfinal"]
score(user_train, user_train.PredictedLogRevenue )
<feature_engineering><EOS> 
user_test['PredictedLogRevenue'] = 0.4 * user_test["lgbfinal"] + 0.4 * user_test["catfinal"] + 0.2 * user_test["xgbfinal"]
user_test[['PredictedLogRevenue']].to_csv('leaky submission.csv', index=True )