Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
import numpy as np
import pandas as pd
import os
import math
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
def determine_first_sale(df):
item_firstsale =(
 df[df["item_cnt_month"] > 0].groupby(by=["item_id", "shop_id"])["date_block_num"].min())
 df["item_firstsale_month"] = df.set_index(["item_id", "shop_id"] ).index.map(item_firstsale)
 df["item_first_sale"] = 0
df.loc[df["item_firstsale_month"] == df["date_block_num"], "item_first_sale"] = 1
df.drop(columns=["item_firstsale_month"], inplace=True)
 return df
df = determine_first_sale(df )<load_from_csv>
test_df = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_df = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_target_df = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 target_cols = train_target_df.columns[1:]
def set_similar_items(df):

 itemstranslated = pd.read_csv(".. /input/filestranslated/items-translated.csv")
 itemdf = pd.DataFrame(
 columns=["item_id", "item_description"], data=itemstranslated.values 
)
 corpus = list(itemdf["item_description"].values)
 vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(corpus)
 cosine_similarities = pd.DataFrame(
 cosine_similarity(X), index=itemdf["item_id"], columns=itemdf["item_id"] 
)
 similar_items = []
for i in cosine_similarities.columns:
similar = np.append(cosine_similarities[i].nlargest(4)[1:].index.values, i)
 similar_items.append(similar)
 similar_items = pd.DataFrame(
 np.vstack(similar_items),
columns=["similar_item1", "similar_item2", "similar_item3", "item_id"], 
)
 sim1 = dict(zip(similar_items["item_id"], similar_items["similar_item1"]))
sim2 = dict(zip(similar_items["item_id"], similar_items["similar_item2"]))
sim3 = dict(zip(similar_items["item_id"], similar_items["similar_item3"]))
df["similar_item1"] = df["item_id"].map(sim1)
 df["similar_item2"] = df["item_id"].map(sim2)
 df["similar_item3"] = df["item_id"].map(sim3)
 return df
df = set_similar_items(df)
<prepare_output>
SEED = 9876
EPOCHS = 30
BATCH_SIZE = 128
FOLDS = 5
REPEATS = 2
LR = 0.0005
N_TARGETS = len(target_cols )
df = downcast_df(df )<categorify>
def seed_everything(seed):
np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
oe = OrdinalEncoder()
df[['item_id','similar_item1','item_cat_group',
'similar_item2','similar_item3','city',
'item_cat_id','shop_id','month','year']] = oe.fit_transform(df[['item_id','similar_item1','item_cat_group',
'similar_item2','similar_item3','city',
'item_cat_id','shop_id','month','year']])
 X_cols_to_drop = [
"train",
"item_cnt_month",
"item_cat_name",
"shop_name",
'binned_item_price'
]<prepare_x_and_y>
def multi_log_loss(y_true, y_pred):
losses = []
for col in y_true.columns:
losses.append(log_loss(y_true.loc[:, col], y_pred.loc[:, col]))
return np.mean(losses )
def df_to_xy(df):
X_train = df[df["date_block_num"] < 33].drop(columns=X_cols_to_drop)
 X_val = df[df["date_block_num"] == 33].drop(columns=X_cols_to_drop)
 y_train = df.loc[df["date_block_num"] < 33, "item_cnt_month"]
y_val = df.loc[df["date_block_num"] == 33, "item_cnt_month"]
X_test = df[df["date_block_num"] == 34].drop(columns=X_cols_to_drop)
 return X_test, X_train, X_val, y_train, y_val
X_test, X_train, X_val, y_train, y_val = df_to_xy(df)
 del df
capture = gc.collect()<train_model>
def preprocess_df(data):
data['cp_type'] =(data['cp_type'] == 'trt_cp' ).astype(int)
 data['cp_dose'] =(data['cp_dose'] == 'D2' ).astype(int)
 return data
model_params = {
"n_estimators": 200,
"num_leaves": 40,
"learning_rate": 0.15,
"min_child_weight": 0.35,
"colsample_bytree": 0.27,
"reg_alpha": 0.83,
"reg_lambda": 0.56,
}
model = LGBMRegressor(**model_params)
 model.fit(
 X_train,
y_train,
eval_metric="rmse",
eval_set=(X_val, y_val),
early_stopping_rounds=10,
categorical_feature=[
"item_id","shop_id",'city',
"date_block_num","item_cat_id",'item_cat_group',
"similar_item1","similar_item2","similar_item3"
] 
)
 pred = model.predict(X_test ).clip(0, 20)
 sub_ID = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")[
"ID"
].values
submissionfile = pd.DataFrame(list(zip(sub_ID, pred)) , columns=["ID", "item_cnt_month"] )<save_to_csv>
x_train = preprocess_df(train_df.drop(columns="sig_id"))
x_test =preprocess_df(test_df.drop(columns="sig_id"))
y_train = train_target_df.drop(columns="sig_id")
 N_FEATURES = x_train.shape[1]
submissionfile.to_csv('submission.csv', index=False )<import_modules>
def create_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.GaussianNoise(NOISE_STD_VECTOR),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(3072, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(3072, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(N_TARGETS, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(lr=LR), sync_period=10), loss='binary_crossentropy', metrics=["accuracy"])
 return model
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import gc
import pickle
import time
from joblib import dump, load
import lightgbm as lgb
from sklearn.metrics import mean_squared_error<import_modules>
def build_train(resume_models = None, repeat_number = 0, folds = 5, skip_folds = 0):
models = []
oof_preds = y_train.copy()
kfold = KFold(folds, shuffle = True)
 for fold,(train_ind, val_ind)in enumerate(kfold.split(x_train)) :
print('-'*50)
 print(f'Training fold {fold + 1}')
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss', factor = 0.4, patience = 3, verbose = 0, min_delta = 0.0001, mode = 'auto')
 checkpoint_path = f'repeat:{repeat_number}_Fold:{fold}.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 model = create_model()
model.fit(x_train.values[train_ind],
y_train.values[train_ind],
validation_data=(x_train.values[val_ind], y_train.values[val_ind]),
callbacks = [cb_lr_schedule, cb_checkpt],
epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=0 
)
 model.load_weights(checkpoint_path)
 oof_preds.loc[val_ind, :] = model.predict(x_train.values[val_ind])
 models.append(model)
 return models, oof_preds
def get_imports() :
for name, val in globals().items() :
if isinstance(val, types.ModuleType):
name = val.__name__.split(".")[0]
elif isinstance(val, type):
name = val.__module__.split(".")[0]
if name == "PIL":
name = "Pillow"
elif name == "sklearn":
name = "scikit-learn"
yield name
imports = list(set(get_imports()))
requirements = []
for m in pkg_resources.working_set:
if m.project_name in imports and m.project_name!="pip":
requirements.append(( m.project_name, m.version))
for r in requirements:
print("{}=={}".format(*r))<load_from_csv>
noise_factors = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]
data = pd.read_pickle('.. /input/eda-preprocessing-feature-engineering/all_data.pkl')
 data = data[data.date_block_num > 5]
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 dropcols = [
"item_cnt_month_lag_12",
"item_cnt_month_lag_12_adv",
"date_item_target_enc_lag_12",
"date_shop_target_enc_lag_12",
"date_city_target_enc_lag_1",
"date_city_target_enc_lag_2",
"date_city_target_enc_lag_3",
"date_type_target_enc_lag_1",
"date_subtype_target_enc_lag_1",
"new_item_cat_avg_lag_1",
"new_item_cat_avg_lag_2",
"new_item_cat_avg_lag_3",
"new_item_shop_cat_avg_lag_1",
"new_item_shop_cat_avg_lag_2",
"new_item_shop_cat_avg_lag_3",
]
X_train = data[data.date_block_num < 33].drop(['item_cnt_month']+dropcols, axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month']+dropcols, axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month']+dropcols, axis=1)
 del data
gc.collect()<init_hyperparams>
feat_stds = x_train[x_train.columns[3:]].std(axis=0 ).values
losses_ind = []
losses_mean = []
losses_mean_adj = []
test_pred_all = []
seed_everything(SEED)
 for j, fac in enumerate(noise_factors):
print('
')
 print('
print(f"Training with Setting {j+1}, Noise Factor: {fac}")
 NOISE_STD_VECTOR = [0, 0, 0]
NOISE_STD_VECTOR.extend(feat_stds * fac)
 oof_preds = []
models = []
for i in range(REPEATS):
print(f"Starting Repeat {i+1}")
 m, oof = build_train(repeat_number = i, folds=FOLDS)
 models = models + m
oof_preds.append(oof)
 print(f"Finished training.")
 mean_oof_preds = y_train.copy()
mean_oof_preds.loc[:, target_cols] = 0
rep_losses = []
print(f"OOF Results for Setting {j+1}, Noise Factor: {fac}")
 for i, p in enumerate(oof_preds):
loss = multi_log_loss(y_train, p)
 print(f"Repeat {i + 1} OOF Log Loss: {loss}")
 rep_losses.append(loss)
 mean_oof_preds.loc[:, target_cols] += p[target_cols]
losses_ind.append(rep_losses)
 mean_oof_preds.loc[:, target_cols] /= len(oof_preds)
 loss_mean = multi_log_loss(y_train, mean_oof_preds)
 print(f"Mean OOF Log Loss: {multi_log_loss(y_train, mean_oof_preds)}")
 losses_mean.append(loss_mean)
 mean_oof_preds.loc[x_train['cp_type'] == 0, target_cols] = 0
loss_mean_adj = multi_log_loss(y_train, mean_oof_preds)
 print(f"Mean OOF Log Loss(ctl adjusted): {loss_mean_adj}
")
 losses_mean_adj.append(loss_mean_adj)
 print(f"Making test predicitons:")
 test_preds = sub.copy()
test_preds[target_cols] = 0
for model in models:
test_preds.loc[:,target_cols] += model.predict(x_test)
 test_preds.loc[:,target_cols] /= len(models)
 test_preds.loc[x_test['cp_type'] == 0, target_cols] = 0
test_pred_all.append(test_preds)
 print("Freeing Memory.")
 for m in models:
del m
K.clear_session()
print('
start_time = time.time()
feature_name = X_train.columns.tolist()
params = {
'objective': 'mse',
'metric': 'rmse',
'num_leaves': 2 ** 7 - 1,
'learning_rate': 0.005,
'feature_fraction': 0.73,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'seed': 0,
'verbose': 1
}
feature_name_indexes = [
'country_part',
'city_code',
'type_code',
'subtype_code',
'category_code',
'item_category_id',
]
lgb_train = lgb.Dataset(X_train[feature_name], Y_train)
 lgb_eval = lgb.Dataset(X_valid[feature_name], Y_valid, reference=lgb_train)
 evals_result = {}
gbm = lgb.train(
 params,
lgb_train,
num_boost_round=3000,
valid_sets=(lgb_train, lgb_eval),
feature_name = feature_name,
categorical_feature = feature_name_indexes,
verbose_eval=5,
evals_result = evals_result,
early_stopping_rounds = 100)
 print(f"Training took {time.time() - start_time} s")
 start_time = time.time()
Y_train_pred = gbm.predict(X_train ).clip(0, 20)
 print(f"Predicting on train set took {time.time() - start_time} s")
 start_time = time.time()
Y_valid_pred = gbm.predict(X_valid ).clip(0, 20)
 print(f"Predicting on valid set took {time.time() - start_time} s")
 print(f"TRAIN RMSE: {round(np.sqrt(mean_squared_error(Y_train, Y_train_pred)) , 5)}, VALID RMSE: {round(np.sqrt(mean_squared_error(Y_valid, Y_valid_pred)) , 5)}")
 dump(gbm, 'lightgbm_model.joblib' )<save_to_csv>
pd.DataFrame({'Noise Factor': noise_factors, 'Loss(each repeat)': losses_ind, 'Losses(mean pred.) ': losses_mean, 'Losses(mean pred.adj.) ': losses_mean_adj} )
start_time = time.time()
Y_test = gbm.predict(X_test ).clip(0, 20)
 print(f"Predicting test set took {time.time() - start_time} s")
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('gbm_y_test.csv', index=False)
 train_preds = pd.DataFrame({
"ID": X_train.index,
"item_cnt_month": Y_train_pred
})
 train_preds.to_csv('gbm_y_train.csv', index=False)
 valid_preds = pd.DataFrame({
"ID": X_valid.index,
"item_cnt_month": Y_valid_pred
})
 valid_preds.to_csv('gbm_y_valid.csv', index=False)
<set_options>
test_preds = sub.copy()
test_preds[target_cols] = 0
for p in test_pred_all:
test_preds.loc[:,target_cols] += p[target_cols]
test_preds.loc[:,target_cols] /= len(test_pred_all)
 test_preds.loc[x_test['cp_type'] == 0, target_cols] = 0
test_preds.to_csv(f'submission.csv', index=False )