Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<prepare_x_and_y>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1)
 del train, test
gc.collect() 
<init_hyperparams>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
params = {'num_leaves': 524,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 124,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47,
}<split>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed)
 scale = "rankgauss"
decompo = "PCA"
ncompo_genes = 150
ncompo_cells = 10
encoding = "dummy"
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=500)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )<save_to_csv>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv', index_col=0)
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col=0)
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv', index_col=0)
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv', index_col=0 )
sub['isFraud'] = y_preds
sub.to_csv("submission.csv", index=False )<save_to_csv>
train_features=train_features.replace('null',np.NaN)
 train_features.isnull().any().sum()
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('Top 50 feature importance over {} folds average'.format(folds.n_splits)) ;<set_options>
train_targets_scored.isnull().any().sum()
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
train_targets_nonscored.isnull().any().sum()
sub_1 = pd.read_csv('/kaggle/input/09522/submission2.csv')
 sub_2 = pd.read_csv('/kaggle/input/gmean-of-low-correlation-lb-0-952x/stack_gmean.csv' )<save_to_csv>
rows = DataFrame(train_features[train_features.loc[:,'cp_type']=='ctl_vehicle'] ).index
drop_id = train_features[train_features.index.isin(rows)].index
drop_id.shape
score_1 = 0.95220
score_2 = 0.95240
sum_scores = score_1 + score_2
weight_1 = score_1/sum_scores
weight_2 = score_2/sum_scores
sub_1['isFraud'] = weight_1*sub_1['isFraud'] + weight_2*sub_2['isFraud']
sub_1.to_csv('submission-.9525.csv', index=False )<set_options>
train_features.drop(['cp_type'], axis=1, inplace = True)
 print(train_features.shape)
 train_features.head()
warnings.filterwarnings('ignore' )<load_from_csv>
train_targets_scored.drop(train_targets_scored[train_targets_scored.index.isin(drop_id)].index, axis=0, inplace=True)
 train_targets_scored.shape
sub_1 = pd.read_csv('.. /input/ieee-simple-lgbm/submission.csv')
 sub_2 = pd.read_csv('.. /input/ieee-cv-options/submission.csv')
 sub_3 = pd.read_csv('.. /input/ieee-lgbm-with-groupkfold-cv/submission.csv')
 sub_4 = pd.read_csv('.. /input/ieee-catboost-baseline-with-groupkfold-cv/submission.csv')
 sub_1['isFraud'] += sub_2['isFraud']
sub_1['isFraud'] += sub_3['isFraud']
sub_1['isFraud'] += sub_4['isFraud']
sub_1.to_csv('submission.csv', index=False )<install_modules>
train_targets_nonscored.drop(train_targets_nonscored[train_targets_nonscored.index.isin(drop_id)].index, axis=0, inplace=True)
 train_targets_nonscored.shape
!pip install catboost -U<load_from_url>
train_outlier_index = train_outlier.index.intersection(trainG_outlier.index ).intersection(trainC_outlier.index)
 train_outlier_index
!git clone https://github.com/lmassaron/deep_learning_for_tabular_data.git<set_options>
test_outlier_index = test_outlier.index.intersection(testG_outlier.index ).intersection(testC_outlier.index)
 test_outlier_index
warnings.filterwarnings("ignore")
<prepare_x_and_y>
train_targets_scored.loc[:,'5-alpha_reductase_inhibitor':].apply(pd.value_counts )
X, Xt = amazon()
y = X["ACTION"].apply(lambda x: 1 if x == 1 else 0 ).values
X.drop(["ACTION"], axis=1, inplace=True )<categorify>
train = train_features.reset_index(drop = True)
 test_all = test_features.reset_index(drop = True)
 test_all.index = range(train.shape[0], test_all.shape[0]+ train.shape[0])
 test_vehicle_index = test_all[test_all['cp_type'] == 'ctl_vehicle'].index
test_noVehicle = test_all[test_all['cp_type'] != 'ctl_vehicle']
test_noVehicle.drop(['cp_type'], axis=1, inplace = True)
 data_all = pd.concat([train, test_noVehicle])
 data_index = data_all.index
data_all = data_all.reset_index(drop = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
label_encoders = [LabelEncoder() for _ in range(X.shape[1])]
for col, column in enumerate(X.columns):
label_encoders[col].fit(X[column].append(Xt[column]))
X[column] = label_encoders[col].transform(X[column])
 Xt[column] = label_encoders[col].transform(Xt[column] )<categorify>
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass
def frequency_encoding(column, df, df_test=None):
frequencies = df[column].value_counts().reset_index()
df_values = df[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).iloc[:,-1].values
if df_test is not None:
df_test_values = df_test[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).fillna(1 ).iloc[:,-1].values
else:
df_test_values = None
return df_values, df_test_values
for column in X.columns:
train_values, test_values = frequency_encoding(column, X, Xt)
 X[column+'_counts'] = train_values
Xt[column+'_counts'] = test_values<define_variables>
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 else:
pass
categorical_variables = [col for col in X.columns if '_counts' not in col]
numeric_variables = [col for col in X.columns if '_counts' in col]<count_unique_values>
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )
X[categorical_variables].nunique()<choose_model_class>
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )
best_params = {'bagging_temperature': 0.6,
'border_count': 200,
'depth': 8,
'iterations': 350,
'l2_leaf_reg': 30,
'learning_rate': 0.30,
'random_strength': 0.01,
'scale_pos_weight': 0.48}
catb = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric = 'AUC',
nan_mode='Min',
thread_count=2,
verbose = False )<choose_model_class>
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )
SEED = 42
FOLDS = 5
skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=SEED )<split>
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )
roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
best_iteration = list()
for train_idx, test_idx in skf.split(X, y):
X_train, y_train = X.iloc[train_idx, :], y[train_idx]
X_test, y_test = X.iloc[test_idx, :], y[test_idx]
train = Pool(data=X_train,
label=y_train,
feature_names=list(X_train.columns),
cat_features=categorical_variables)
 test = Pool(data=X_test,
label=y_test,
feature_names=list(X_test.columns),
cat_features=categorical_variables)
 catb.fit(train,
verbose_eval=100,
early_stopping_rounds=50,
eval_set=test,
use_best_model=True,
plot=False)
 best_iteration.append(catb.best_iteration_)
 preds = catb.predict_proba(X_test)
 oof[test_idx] = preds[:,1]
roc_auc.append(roc_auc_score(y_true=y_test, y_score=preds[:,1]))
average_precision.append(average_precision_score(y_true=y_test, y_score=preds[:,1]))<compute_test_metric>
train_df = data_all[: train.shape[0]]
test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True)
 test_noVehicle_index = data_index[train_df.shape[0]:]
test_noVehicle_index = list(map(lambda x:x - train_df.shape[0], test_noVehicle_index))
test_vehicle_index = list(map(lambda x:x - train_df.shape[0], test_vehicle_index))
targets = targets_cp_train.reset_index(drop = True )
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof)) 
<choose_model_class>
MAX_EPOCH = 200
N_SPLITS = 10
Lr = 2e-2
Weight_decay = 1e-5
Nd = 32
Na = 32
Steps = 1
Batch = 1024
best_params = {'bagging_temperature': 0.6,
'border_count': 200,
'depth': 8,
'iterations': int(np.median(best_iteration)* 1.3),
'l2_leaf_reg': 30,
'learning_rate': 0.30,
'random_strength': 0.01,
'scale_pos_weight': 0.48}
catb = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric = 'AUC',
nan_mode='Min',
thread_count=2,
verbose = False)
 train = Pool(data=X,
label=y,
feature_names=list(X_train.columns),
cat_features=categorical_variables)
 catb.fit(train,
verbose_eval=100,
plot=False)
 submission = pd.DataFrame(Xt.id)
 Xt_pool = Pool(data=Xt[list(X_train.columns)],
feature_names=list(X_train.columns),
cat_features=categorical_variables)
 submission['Action'] = catb.predict_proba(Xt_pool)[:,1]
submission.to_csv("catboost_submission.csv", index=False)
 cat_boost_submission = submission.copy()<import_modules>
tabnet_params = dict(
 n_d = Nd,
n_a = Na,
n_steps = Steps,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = Lr, weight_decay = Weight_decay),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
import tensorflow as tf
from tensorflow.keras import backend as K
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam, Nadam
from tensorflow.keras.layers import Input, Embedding, Reshape, GlobalAveragePooling1D
from tensorflow.keras.layers import Flatten, concatenate, Concatenate, Lambda, Dropout, SpatialDropout1D
from tensorflow.keras.layers import Reshape, MaxPooling1D,BatchNormalization, AveragePooling1D, Conv1D
from tensorflow.keras.layers import Activation, LeakyReLU
from tensorflow.keras.optimizers import SGD, Adam, Nadam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.regularizers import l2, l1_l2
from keras.losses import binary_crossentropy
from sklearn.metrics import roc_auc_score
from sklearn.metrics import average_precision_score
import matplotlib.pyplot as plt<import_modules>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux = 0.5*(1 - y_true)* np.log(1 - logits + 1e-15)+ 0.5*y_true * np.log(logits + 1e-15)
 wAux = 0.005 *(1 - y_true)* np.log(1 - logits + 1e-15)+ 0.995 * y_true * np.log(logits + 1e-15)
 return 0.7 * np.mean(-aux)+ 0.3 * np.mean(-wAux )
get_custom_objects().update({'gelu': Activation(gelu)})
 get_custom_objects().update({'mish': Mish(mish)})
 get_custom_objects().update({'leaky-relu': Activation(LeakyReLU(alpha=0.2)) } )<choose_model_class>
scores_auc_all = []
test_cv_preds = []
mskf = MultilabelStratifiedKFold(n_splits = N_SPLITS, random_state = 0, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = Batch,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
def tabular_dnn(numeric_variables, categorical_variables, categorical_counts,
feature_selection_dropout=0.2, categorical_dropout=0.1,
first_dense = 256, second_dense = 256, dense_dropout = 0.2,
activation_type=gelu):
numerical_inputs = Input(shape=(len(numeric_variables),))
numerical_normalization = BatchNormalization()(numerical_inputs)
 numerical_feature_selection = Dropout(feature_selection_dropout )(numerical_normalization)
 categorical_inputs = []
categorical_embeddings = []
for category in categorical_variables:
categorical_inputs.append(Input(shape=[1], name=category))
category_counts = categorical_counts[category]
categorical_embeddings.append(
 Embedding(category_counts+1,
int(np.log1p(category_counts)+1),
name = category + "_embed" )(categorical_inputs[-1]))
categorical_logits = Concatenate(name = "categorical_conc" )([Flatten()(SpatialDropout1D(categorical_dropout )(cat_emb))
for cat_emb in categorical_embeddings])
 x = concatenate([numerical_feature_selection, categorical_logits])
 x = Dense(first_dense, activation=activation_type )(x)
 x = Dropout(dense_dropout )(x)
 x = Dense(second_dense, activation=activation_type )(x)
 x = Dropout(dense_dropout )(x)
 output = Dense(1, activation="sigmoid" )(x)
 model = Model([numerical_inputs] + categorical_inputs, output)
 return model<define_variables>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
SEED = 42
FOLDS = 5
BATCH_SIZE = 512<choose_model_class>
tgt_col = [col for col in sample_submission.columns if col not in ["sig_id"]]
test_predicts = sample_submission.copy()
test_predicts[tgt_col] = 0
test_predicts.loc[test_noVehicle_index,tgt_col] = test_preds_all.mean(axis = 0)
 test_predicts.to_csv('submission.csv', index=False)
 test_predicts.head()