Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<count_values>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
Counter(output )<import_modules>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
print(os.listdir(".. /input")) 
<load_from_csv>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
def get_cat_val(X, arr, cat):
cat_list = []
for ind in arr:
cat_list.append(X[ind, cat])
 return cat_list
def get_dog_val(X, arr, dog):
dog_list = []
for ind in arr:
dog_list.append(X[ind, dog])
 return dog_list
def weighted_average(item):
try:
return np.average(item, weights=np.arange(len(item), 0, -1))
except ZeroDivisionError:
return 0
def get_train_meta() :
x_train = pd.read_csv(".. /input/train/train.csv")
 label_annotations = collections.defaultdict(list)
 label_scores = collections.defaultdict(list)
 for file in os.listdir('.. /input/train_metadata/'):
tmp = file.split() [0].split('.json')[0]
key, val = tmp.split('-')
 try:
with open('.. /input/train_metadata/'+file)as f:
data = json.load(f)
 if data.get('labelAnnotations'):
for element in data['labelAnnotations']:
label = element['description']
score = element['score']
label_annotations[key].append(label)
 label_scores[key].append(score)
 else:
label_annotations[key].append('N/A')
 label_scores[key].append(-1)
 except FileNotFoundError:
print('Oopsie')
 x_train['label_annotation'] = x_train['PetID'].map(label_annotations)
 x_train['label_score'] = x_train['PetID'].map(label_scores)
 x_train['label_annotation'] = [[word_tokenize(word)for word in sentence] for sentence in x_train['label_annotation']]
flatten = lambda l: [item for sublist in l for item in sublist]
x_train['label_annotation'] = x_train['label_annotation'].map(flatten)
 x_train['label_annotation'] = [[stem(word)for word in sentence] for sentence in x_train['label_annotation']]
list_to_string = lambda l: ' '.join(l)
 x_train['label_annotation'] = x_train['label_annotation'].map(list_to_string)
 vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(x_train['label_annotation'])
 ind = x_train.index.values
cat_val = get_cat_val(X, ind, vectorizer.vocabulary_['cat'])
 dog_val = get_dog_val(X, ind, vectorizer.vocabulary_['dog'])
 x_train.loc[:, 'label_cat'] = cat_val
x_train.loc[:, 'label_dog'] = dog_val
x_train = x_train.drop(columns=['label_annotation'])
 x_train['label_score'] = x_train['label_score'].map(weighted_average)
 return x_train
def get_test_meta() :
x_test = pd.read_csv(".. /input/test/test.csv")
 label_annotations = collections.defaultdict(list)
 label_scores = collections.defaultdict(list)
 for file in os.listdir('.. /input/test_metadata/'):
tmp = file.split() [0].split('.json')[0]
key, val = tmp.split('-')
 try:
with open('.. /input/test_metadata/'+file)as f:
data = json.load(f)
 if data.get('labelAnnotations'):
for element in data['labelAnnotations']:
label = element['description']
score = element['score']
label_annotations[key].append(label)
 label_scores[key].append(score)
 else:
label_annotations[key].append('N/A')
 label_scores[key].append(-1)
 except FileNotFoundError:
print('Oopsie')
 x_test['label_annotation'] = x_test['PetID'].map(label_annotations)
 x_test['label_score'] = x_test['PetID'].map(label_scores)
 x_test['label_annotation'] = [[word_tokenize(word)for word in sentence] for sentence in x_test['label_annotation']]
flatten = lambda l: [item for sublist in l for item in sublist]
x_test['label_annotation'] = x_test['label_annotation'].map(flatten)
 x_test['label_annotation'] = [[stem(word)for word in sentence] for sentence in x_test['label_annotation']]
list_to_string = lambda l: ' '.join(l)
 x_test['label_annotation'] = x_test['label_annotation'].map(list_to_string)
 vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(x_test['label_annotation'])
 ind = x_test.index.values
cat_val = get_cat_val(X, ind, vectorizer.vocabulary_['cat'])
 dog_val = get_dog_val(X, ind, vectorizer.vocabulary_['dog'])
 x_test.loc[:, 'label_cat'] = cat_val
x_test.loc[:, 'label_dog'] = dog_val
x_test = x_test.drop(columns=['label_annotation'])
 x_test['label_score'] = x_test['label_score'].map(weighted_average)
 return x_test 
<load_from_csv>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
<feature_engineering>
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )
train = get_train_meta() 
<categorify>
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )
test = get_test_meta()<string_transform>
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
def extract_text_features(text):
tp = type(text)is str
if not tp:
text = "fsdfl"
features = []
header = []
header.append("PetID")
 bag_of_words = nltk.word_tokenize(text)
 header.append("Amount of Words")
 features.append(len(bag_of_words))
sent_text = nltk.sent_tokenize(text)
 header.append("Amount of Sentences")
 features.append(len(sent_text))
if not tp:
features = list(np.zeros(len(header)-1))
return header, features
def get_train_text_and_name_features() :
training_df = pd.read_csv('.. /input/train/train.csv')
 text_data = training_df[["PetID","Description","Name","AdoptionSpeed"]]
text_features_table = []
name_features_table = []
counter = 0
for(PID,profile,name,a_s)in text_data.values:
header,features = extract_text_features(profile)
 text_features_table.append([PID]+features)
 text_features_pd = pd.DataFrame(text_features_table,columns=header)
 return text_features_pd
def get_test_text_and_name_features() :
test_df = pd.read_csv('.. /input/test/test.csv')
 text_data = test_df[["PetID","Description","Name"]]
text_features_table = []
name_features_table = []
counter = 0
for(PID,profile,name)in text_data.values:
header,features = extract_text_features(profile)
 text_features_table.append([PID]+features)
 text_features_pd = pd.DataFrame(text_features_table,columns=header)
 return text_features_pd<feature_engineering>
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
print(np.where(mask))
print(len(cols_numeric))
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
des_train = get_train_text_and_name_features() 
<data_type_conversions>
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
gaussrank = scaler.fit(data_all[cols_numeric])
 with open("gaussrank.pkl", 'wb')as f:
pickle.dump(gaussrank, f, pickle.HIGHEST_PROTOCOL)
 f.close()
data_all[cols_numeric] = scaler.transform(data_all[cols_numeric])
 else:
pass
train['PetID']=train['PetID'].astype(str)
 test['PetID']=test['PetID'].astype(str )<feature_engineering>
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes,
random_state = seed ).fit(data_all[GENES])
 with open("pcas_genes.pkl", 'wb')as f1:
pickle.dump(pca_genes, f1, pickle.HIGHEST_PROTOCOL)
 f1.close()
pca_cells = PCA(n_components = ncompo_cells,
random_state = seed ).fit(data_all[CELLS])
 with open("pcas_cells.pkl", 'wb')as f2:
pickle.dump(pca_cells, f2, pickle.HIGHEST_PROTOCOL)
 f2.close()
else:
pass
des_test = get_test_text_and_name_features()<create_dataframe>
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )
AoW_train = pd.DataFrame(des_train['Amount of Words'])
 AoW_test = pd.DataFrame(des_test['Amount of Words'] )<merge>
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )
x_train = train.join(AoW_train)
 x_test = test.join(AoW_test )<drop_column>
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )
def extract_features(X):
X_features = X.drop(["Name","RescuerID","Description","PetID"],axis=1)
 return X_features
x_train = extract_features(x_train)
 x_testt = extract_features(x_test )<feature_engineering>
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )
X_mean = x_train.Fee.mean(axis=0)
 X_std = x_train.Fee.std(axis=0)
 x_train.Fee =(x_train.Fee-X_mean)/X_std
x_testt.Fee =(x_train.Fee-X_mean)/X_std<prepare_x_and_y>
features_to_drop = ["sig_id", "cp_type"]
data_all.drop(features_to_drop, axis = 1, inplace = True)
 try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )
y_train = train['AdoptionSpeed'].values<import_modules>
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold 
<split>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'silent': 1,
}
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test 
<train_on_grid>
scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 0, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 model.save_model(f'fold_{fold_nb}.pth')
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 save('oof_tabnet.npy', oof_preds_all)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<compute_train_metric>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
<train_on_grid><EOS> 
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()