Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<rename_columns>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
df_train = agg_pd.T.reset_index()
df_train = df_train.set_index("id")
 rename_dict = {}
for c in df_train.columns:
if c not in ['level', 'state_id', 'store_id', 'cat_id', 'dept_id', 'item_id']:
rename_dict[c] = "d_%s" % c
df_train.rename(columns=rename_dict, inplace=True)
 day_cols = pd.Series([c for c in df_train.columns if c not in ['level', 'state_id', 'store_id', 'cat_id', 'dept_id', 'item_id']])
 print(df_train.shape)
 df_train.head()<feature_engineering>
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
df_calendar = pd.read_csv(CALENDAR)
 df_calendar.index = df_calendar['d'].values
df_calendar['ds'] = pd.to_datetime(df_calendar['date'])
 df_calendar['quarter'] = df_calendar['ds'].dt.quarter
df_calendar.head()<create_dataframe>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
events1 = pd.Series(df_calendar['event_name_1'].values, index=df_calendar['ds'].values ).dropna()
events2 = pd.Series(df_calendar['event_name_2'].values, index=df_calendar['ds'].values ).dropna()
holidays = pd.DataFrame(pd.concat([events1, events2], axis=0))
holidays['ds'] = holidays.index.values
holidays.rename({0: 'holiday'}, axis=1, inplace=True)
 holidays.reset_index(drop=True, inplace=True)
 del events1, events2
holidays.head()<feature_engineering>
data_path = ".. /input/lish-moa/"
train = pd.read_csv(data_path+'train_features.csv')
 train.drop(columns=["sig_id"], inplace=True)
 train_targets_scored = pd.read_csv(data_path+'train_targets_scored.csv')
 train_targets_scored.drop(columns=["sig_id"], inplace=True)
 test = pd.read_csv(data_path+'test_features.csv')
 test.drop(columns=["sig_id"], inplace=True)
 submission = pd.read_csv(data_path+'sample_submission.csv')
 remove_vehicle = False
if remove_vehicle:
kept_index = train['cp_type']=='trt_cp'
train = train.loc[kept_index].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[kept_index].reset_index(drop=True)
 train["cp_type"] =(train["cp_type"]=="trt_cp")+ 0
train["cp_dose"] =(train["cp_dose"]=="D1")+ 0
test["cp_type"] =(test["cp_type"]=="trt_cp")+ 0
test["cp_dose"] =(test["cp_dose"]=="D1")+ 0
X_test = test.values
def clean_data(df_train, day_cols, indx):
t = df_train.loc[indx].copy()
t.loc[day_cols[(( t.loc[day_cols]>0 ).cumsum() ==0 ).values]] = np.nan
q1 = t.loc[day_cols].quantile(0.25)
 q3 = t.loc[day_cols].quantile(0.75)
 iqr = q3-q1
qm =(q3+1.5*iqr)
 t.loc[day_cols][t.loc[day_cols]>qm] = qm
return t<define_variables>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit
from sklearn.metrics import roc_auc_score
future_preds = 28
day_fit_cols = day_cols<define_variables>
MAX_EPOCH=200
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10,
)
ignore_dates = ["2011-12-25", "2012-12-25", "2013-12-25", "2014-12-25", "2015-12-25",
"2011-12-24", "2012-12-24", "2013-12-24", "2014-12-24", "2015-12-24",
"2011-12-31", "2012-12-31", "2013-12-31", "2014-12-31", "2015-12-31",
"2012-01-01", "2013-01-01", "2014-01-01", "2015-01-01", "2016-01-01"]
ignore_days = []<predict_on_test>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux )
def make_prediction(indx, model_columns = 'yhat', ret_columns = 'yhat', full_predict = True):
global df_train, holidays, df_calendar
changepoints=list()
uncertainty_samples=False
changepoint_prior_scale=0.1
changepoint_range=0.9
n_changepoints=50
holidays_prior_scale=10
yearly_seasonality=10
weekly_seasonality=3
daily_seasonality=False
monthly_fourier_order=8
quarter_fourier_order=6
seasonality_prior_scale=10
seasonality_mode = 'multiplicative'
target = df_train.loc[indx, day_fit_cols]
snap_state_id = str(df_train.loc[indx, 'state_id'])
 cols = ['ds', 'month', 'wday', 'quarter']
if snap_state_id in ["CA", "TX", "WI"]:
cols = cols + ['snap_'+snap_state_id]
df = df_calendar.iloc[:target.shape[0]+future_preds][cols].copy()
df['y'] = target
df['ft_lag28'] = df['y'].shift(28)
 df["ft_lag28"].fillna(method ='bfill', inplace = True)
 df['ft_lag35'] = df['y'].shift(35)
 df["ft_lag35"].fillna(method ='bfill', inplace = True)
 df['ft_lag42'] = df['y'].shift(42)
 df["ft_lag42"].fillna(method ='bfill', inplace = True)
 df['ft_lag28_roll28_std'] = df['y'].shift(28 ).rolling(28 ).std()
df["ft_lag28_roll28_std"].fillna(method ='bfill', inplace = True)
 df['ft_lag28_roll28_max'] = df['y'].shift(28 ).rolling(28 ).max()
df["ft_lag28_roll28_max"].fillna(method ='bfill', inplace = True)
 df['ft_lag28_roll28_min'] = df['y'].shift(28 ).rolling(28 ).min()
df["ft_lag28_roll28_min"].fillna(method ='bfill', inplace = True)
 GROWTH = 'linear'
m = Prophet(growth=GROWTH, uncertainty_samples=uncertainty_samples, changepoint_prior_scale=changepoint_prior_scale, changepoint_range=changepoint_range,
n_changepoints = n_changepoints,
holidays_prior_scale=holidays_prior_scale, yearly_seasonality=yearly_seasonality,
daily_seasonality=daily_seasonality, weekly_seasonality=weekly_seasonality,
holidays=holidays, seasonality_mode=seasonality_mode, seasonality_prior_scale=seasonality_prior_scale)
 m.add_country_holidays(country_name='US')
 if not monthly_fourier_order is None:
m.add_seasonality(name='monthly', period=365.25/12, fourier_order=monthly_fourier_order)
 if not quarter_fourier_order is None:
m.add_seasonality(name='quarterly', period=365.25/4, fourier_order=quarter_fourier_order)
 for reg in df.columns:
if reg!='ds' and reg!='y':
m.add_regressor(reg)
 target_first_valid_index = target.first_valid_index()
if GROWTH == "logistic":
df["cap"] = df["y"].max()
df["floor"] = df["y"].min()
m.fit(df.loc[target.loc[target_first_valid_index:].index])
 if 'y' not in ret_columns:
df.drop(['y'], axis=1, inplace=True)
 res = None
if full_predict == True:
forecast = m.predict(df.loc[target_first_valid_index:])
 forecast["yhat"] = forecast["yhat"].astype(np.float32)
 res = forecast[model_columns]
res.index = df.loc[target_first_valid_index:].index.values
res = df.merge(res, left_index=True, right_index=True, how="left")[ret_columns]
else:
forecast = m.predict(df.iloc[-future_preds:])
 forecast["yhat"] = forecast["yhat"].astype(np.float32)
 res = forecast[model_columns]
res.index = df.iloc[-future_preds:].index.values
return(indx, res )<drop_column>
scores_auc_all= []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, random_state=0, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, train_targets_scored)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train.values[train_idx, :], train_targets_scored.values[train_idx, :]
X_val, y_val = train.values[val_idx, :], train_targets_scored.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
train_indxs = ["Total_X"]
train_indxs = train_indxs + ["CA_X", "TX_X", "WI_X"]
train_indxs = train_indxs + ['CA_1_X', 'CA_2_X', 'CA_3_X','CA_4_X', 'TX_1_X', 'TX_2_X', 'TX_3_X', 'WI_1_X', 'WI_2_X', 'WI_3_X']
train_indxs = train_indxs + ['HOBBIES_X', 'FOODS_X', 'HOUSEHOLD_X']
train_indxs = train_indxs + ['HOBBIES_1_X', 'HOBBIES_2_X', 'FOODS_1_X', 'FOODS_2_X', 'FOODS_3_X', 'HOUSEHOLD_1_X', 'HOUSEHOLD_2_X']
m_score = 0
for train_ind in train_indxs: 
(_, pred)= make_prediction(train_ind, model_columns = ['yhat'], ret_columns = ['yhat', 'ds', 'y'])
 fig, ax = plt.subplots(figsize=(DEFAULT_FIG_WIDTH, 4))
d = pred.set_index("ds" ).plot(kind="line", y=["y", "yhat"], ax=ax, linestyle='-', linewidth=0.8)
 score = np.log1p(mean_squared_error(pred["y"][:-28], pred["yhat"][:-28]))
m_score = m_score +(score / len(train_indxs))
plt.title("%s LRMSE=%.3f" %(train_ind, score))
plt.show()
print("Average LRMSE=%.3f" % m_score )<predict_on_test>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )
print('Predicting...', flush=True)
 start_time = time.time()
train_indxs = df_train.index
res = []
for train_indx in tqdm(train_indxs):
r = make_prediction(train_indx)
 res.append(r)
 end_time = time.time()
print('Exec speed=%.2f' %(( end_time-start_time)/train_indxs.shape[0]))<prepare_output>
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
submission[all_feat] = test_preds_all.mean(axis=0)
 submission.loc[test['cp_type']==0, submission.columns[1:]] = 0
submission.to_csv('submission.csv', index=None )