Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<merge>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
data = data.merge(shops_data, on = 'shop_id', how = 'left' ).fillna(0)
 data = data.merge(items_data, on = 'item_id', how = 'left' ).fillna(0 )<merge>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
target_mean = data.groupby(col_dsicat, as_index = False)['target'].mean()
target_mean.columns = col_dsicat + ['target_meansicat']
data = pd.merge(data, target_mean, on = col_dsicat, how = 'left')
 lags_ = [1]
data = lags(data, lags_, 'target_meansicat')
 data = data.drop('target_meansicat', axis = 1 )<data_type_conversions>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
data ['inactive_items'] = data ['item_id'].isin(inactive_items ).replace({False:0, True:1} ).astype(np.int16)
 data = data.merge(holidays, on = 'date_block_num', how = 'left')
 bins = np.linspace(data.price.min() , data.price.max() ,6)
 labels = ['very_low', 'low', 'medium', 'high','very_high']
data ['price_bins'] = pd.cut(data ['price'],bins,labels = labels, include_lowest = True)
 data ['price_bins'] = data ['price_bins'].astype('O')
 data ['price_bins'] = LabelEncoder().fit_transform(data ['price_bins'] ).astype(np.int8 )<data_type_conversions>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
data ['month'] = data ['date_block_num'].apply(lambda x : x%12+1 ).astype(np.int8)
 data ['year'] = data ['date_block_num'].apply(lambda x : x//12 ).astype(np.int8 )<categorify>
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 print(train.shape,test.shape)
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 print(train.shape,test.shape)
 train_col_size = train.shape[0]
cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 print(train.shape,test.shape)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
print(train.shape,test.shape)
 if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
print(train.shape,test.shape)
 if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
print(train.shape,test.shape)
 for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 print(train.shape,test.shape)
 features_to_drop = ["sig_id", "cp_type"]
train.drop(features_to_drop, axis = 1, inplace = True)
 test.drop(features_to_drop, axis = 1, inplace = True)
 try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
print(train.shape,test.shape)
 train_df = train[: train_col_size]
train_df.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 X_test = test.values
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 data['days'] = data['month'].map(days)
 data['days'] = data['days'].astype(np.float16 )<create_dataframe>
req_data = data.copy()<drop_column>
req_data = req_data.drop(['item_category_id','price', 'revenue', 'shop_id', 'item_id'], axis = 1 ).fillna(0)
 req_data = req_data.loc [req_data.date_block_num > 11, :]<set_options>
del data
gc.collect()
del grid
gc.collect()
del target_mean
gc.collect()
del shops_data
gc.collect()
del items_data
gc.collect()
del target
gc.collect()
del price
gc.collect()
del train_data
gc.collect()
del item_cat_data
gc.collect()
del holidays
gc.collect()
del revenue
gc.collect()<prepare_x_and_y>
X = req_data.loc [(req_data.date_block_num<33),:].drop(['date_block_num', 'target'], axis = 1)
 y = req_data.loc [(req_data.date_block_num<33),:]['target'].clip(0,20)
 X_valid = req_data [req_data ['date_block_num']==33].drop(['date_block_num', 'target'], axis = 1)
 y_valid = req_data [req_data ['date_block_num']==33]['target'].clip(0,20)
 X_test = req_data [req_data ['date_block_num']==34].drop(['date_block_num', 'target'], axis = 1 )<set_options>
del req_data
gc.collect() ;<train_model>
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
model = XGBRegressor(
 max_depth=8,
n_estimators=150,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42, reg_lambda=1, gamma = 0)
 model.fit(
 X,
y,
eval_metric="rmse",
eval_set=[(X, y),(X_valid, y_valid)],
verbose=True,
early_stopping_rounds = 6 )<predict_on_test>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )
prediction1 = model.predict(X_valid ).clip(0,20)
 prediction2 = model.predict(X_test ).clip(0,20 )<compute_test_metric>
scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 7
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
SEED =(42,43,44)
 for s in SEED:
tabnet_params['seed'] = s
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = s, shuffle = True)
 for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 print(saving_path_name)
 saving_path_name = 'TabNet_seed_'+str(tabnet_params['seed'])+'_fold_'+str(fold_nb+1)
 saved_filepath = model.save_model(saving_path_name)
 print(saved_filepath)
 input('Press enter to continue')
 loaded_model = TabNetRegressor()
loaded_model.load_model(saving_path_name+'.zip')
 preds_val = loaded_model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
Score = sqrt(mean_squared_error(prediction1, y_valid))
print(round(np.mean(Score),4))<create_dataframe>
saving_path_name = 'TabNet_seed_'+str(tabnet_params['seed'])+'_fold_'+str(fold_nb+1)
 loaded_model = TabNetRegressor()
loaded_model.load_model(saved_filepath)
 preds_train = loaded_model.predict_proba(X_train)
 print(preds_train[:,1] )
df = pd.DataFrame(model.get_booster().get_score(importance_type='weight'), index =[0] ).transpose()
df.columns = ['features']<compute_test_metric>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
<save_to_csv><EOS> 
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()