Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<prepare_x_and_y>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))<categorify>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelcat = labelcat.loc[labelcat.label_id.isin(applabels.label_id.unique())]
labelencoder = LabelEncoder().fit(labelcat.category)
 labelcat['label'] = labelencoder.transform(labelcat.category)
 nlabels = len(labelencoder.classes_)
 print('number of unique labels:',nlabels)
 print('recoded label categories', '/n',labelcat.head(n=20))
applabels=applabels.merge(labelcat[['label','label_id']],
how='left',left_on='label_id',right_on='label_id')
 devicelabels =(deviceapps[['device_id','app']] 
.merge(applabels[['app','label']])
.groupby(['device_id','label'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 devicelabels.head()<correct_missing_values>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
d = devicelabels.dropna(subset=['trainrow'])
 Xtr_label = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.label)) ,
shape=(gatrain.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
 Xte_label = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.label)) ,
shape=(gatest.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))<concatenate>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = ["PCA", "ICA", "FactorAnalysis"]
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label,
Xtr_tod, Xtr_dow, Xtr_lat, Xtr_lng), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label,
Xte_tod, Xte_dow, Xte_lat, Xte_lng), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))<categorify>
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )
targetencoder = LabelEncoder().fit(gatrain.group)
 y = targetencoder.transform(gatrain.group )<init_hyperparams>
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )
params = {}
params['booster'] = 'gblinear'
params['objective'] = "multi:softprob"
params['eval_metric'] = 'mlogloss'
params['eta'] = 0.005
params['num_class'] = 12
params['lambda'] = 3
params['alpha'] = 2<train_model>
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
kf = list(StratifiedKFold(y, n_folds=10, shuffle=True, random_state=4242)) [0]
Xtr, Xte = Xtrain[kf[0], :], Xtrain[kf[1], :]
ytr, yte = y[kf[0]], y[kf[1]]
print('Training set: ' + str(Xtr.shape))
print('Validation set: ' + str(Xte.shape))
d_train = xgb.DMatrix(Xtr, label=ytr)
 d_valid = xgb.DMatrix(Xte, label=yte)
 watchlist = [(d_train, 'train'),(d_valid, 'eval')]<save_to_csv>
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
clf = xgb.train(params, d_train, 1000, watchlist, early_stopping_rounds=25)
 pred = clf.predict(xgb.DMatrix(Xtest))
pred = pd.DataFrame(pred, index = gatest.index, columns=targetencoder.classes_)
 pred.head()
pred.to_csv('sparse_xgb_v11.csv', index=True )<load_from_csv>
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass
df_train = pd.read_csv('/kaggle/input/mercari-price-suggestion-challenge/train.tsv', delimiter='\t')
 df_test = pd.read_csv('/kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv', delimiter='\t' )<concatenate>
if "PCA" in decompo:
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 if "TSNE" in decompo :
print(b_, "TSNE")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = TSNE(n_components = ncompo_genes, n_iter=900, method='exact' ).fit_transform(data_all[GENES])
 pca_cells = TSNE(n_components = ncompo_cells, n_iter=900, method='exact' ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 if "ICA" in decompo:
print(b_, "ICA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = FastICA(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = FastICA(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"ica_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"ica_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 if "FactorAnalysis" in decompo:
print(b_, "FactorAnalysis")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = FactorAnalysis(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = FactorAnalysis(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"factor_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"factor_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
df = pd.concat([df_train, df_test], axis=0)
 df.head()<define_variables>
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )
nrow_train=df_train.shape[0]<prepare_x_and_y>
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )
y_train = np.log1p(df_train["price"] )<set_options>
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )
del df_train
gc.collect()<prepare_x_and_y>
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )
count = CountVectorizer(min_df=10)
 X_name = count.fit_transform(df["name"] )<data_type_conversions>
features_to_drop = ["sig_id", "cp_type"]
data_all.drop(features_to_drop, axis = 1, inplace = True)
 try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )
count_category = CountVectorizer()
df["category_name"] = df["category_name"].fillna("Other" ).astype("category")
 X_category = count_category.fit_transform(df["category_name"] )<categorify>
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
count_descp = TfidfVectorizer(max_features = 50000,ngram_range =(1,3),stop_words = "english")
 df["item_description"] = df["item_description"].fillna("None")
 X_descp = count_descp.fit_transform(df["item_description"] )<feature_engineering>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )
df["brand_name"] = df["brand_name"].fillna("unknown")
 pop_brands = df["brand_name"].value_counts().index[:2500]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"
df["brand_name"] = df["brand_name"].astype("category")
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df["brand_name"] )<import_modules>
scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 0, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
import scipy<categorify>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
<concatenate><EOS> 
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()