Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<load_from_csv>
os.environ['OMP_NUM_THREADS'] = '4'
print(os.listdir(".. /input"))
targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' )<load_from_csv>
stops = set(stopwords.words('english'))
def make_folds(folds = 5, random_state = 0, stratify = True, scored = None):
drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 if scored is None:
scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 targets = scored.columns[1:]
scored = scored.merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
if stratify:
skf = MultilabelStratifiedKFold(n_splits=folds, shuffle=True, random_state=random_state)
 else:
skf = KFold(n_splits=folds, shuffle=True, random_state=random_state)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 if stratify:
skf = MultilabelStratifiedKFold(n_splits=folds, shuffle=True, random_state=random_state)
 else:
skf = KFold(n_splits=folds, shuffle=True, random_state=random_state)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = np.nan
scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] = scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored[['sig_id','fold']].copy()<feature_engineering>
def standardize_text(df, text_field):
df[text_field] = df[text_field].str.replace(r"http\S+", "")
 df[text_field] = df[text_field].str.replace(r"http", "")
 df[text_field] = df[text_field].str.replace(r"@\S+", "")
 df[text_field] = df[text_field].str.replace(r"[^A-Za-z0-9() ,!?@'\`"\_
]", " ")
 df[text_field] = df[text_field].str.replace(r"@", "at")
 df[text_field] = df[text_field].str.lower()
return df
FOLDS = 10; SEED = 42
ff = make_folds(folds=FOLDS, random_state=SEED, stratify=True, scored=targets)
 trn_df['kfold'] = ff.fold.values<create_dataframe>
def removing_stopwords(df, text_field):
for i in range(len(df[text_field])) :
df[text_field][i] = ' '.join([word for word in df[text_field][i].split() if word not in stops])
 return df
df = trn_df.copy()<filter>
EMBEDDING_FILE = '.. /input/fast-text-vector/crawl-300d-2M.vec'
sig_ids = test_features[test_features['cp_type'] == 'ctl_vehicle']['sig_id'].values<define_variables>
train = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/test.csv" )
cat_names = ['cp_type', 'cp_time', 'cp_dose']
cont_names = [c for c in train_features.columns if c not in cat_names and c != 'sig_id']<load_pretrained>
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 train = removing_stopwords(train,"comment_text")
 test = removing_stopwords(test,"comment_text" )
def get_data(fold):
val_idx = df[df.kfold==fold].index
dls = TabularDataLoaders.from_df(df, path=path, y_names=cols,
cat_names = cat_names,
cont_names = cont_names,
procs = [Categorify, FillMissing, Normalize],
valid_idx=val_idx,
bs=64)
 return dls 
<train_on_grid>
train = standardize_text(train,"comment_text")
 test = standardize_text(test,"comment_text" )
test_sc = []
for i in tqdm(range(FOLDS)) :
dls = get_data(i)
 learn = tabular_learner(dls , y_range=(0,1), layers=[1024, 512, 512, 256], loss_func = BCELossFlat() , model_dir='/kaggle/working/')
 name = 'best_model_' + str(i)
 cb = SaveModelCallback(monitor='valid_loss',fname=name ,mode='min')
 lr = 9e-3
learn.fit_one_cycle(10, slice(lr/(2.6**4),lr),cbs=cb)
 learn.load(name)
 test_dl = learn.dls.test_dl(test_features)
 sub = learn.get_preds(dl=test_dl)
 test_sc.append(sub[0].numpy())
 learn.export('/kaggle/working/'+name+'.pkl')
 test_sc = np.array(test_sc )<feature_engineering>
train_x = train["comment_text"].values
train_y = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
test_x = test["comment_text"].values
avg_prds = test_sc.mean(axis=0 )<prepare_output>
max_features = 30000
maxlen = 100
embed_size = 300
submission = sample_submission.copy()
submission[cols] = avg_prds
submission.loc[submission['sig_id'].isin(test_features.loc[test_features['cp_type'] =='ctl_vehicle', 'sig_id']), train_targets_scored.columns[1:]] = 0
submission['atp-sensitive_potassium_channel_antagonist'] = 0
submission['erbb2_inhibitor'] = 0<save_to_csv>
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x)+ list(test_x))
submission.to_csv('submission_tabular.csv',index=False )<import_modules>
train_x = tokenizer.texts_to_sequences(train_x)
 test_x = tokenizer.texts_to_sequences(test_x )
from fastai.basics import *
from pytorch_tabnet import *
from fast_tabnet.core import *<choose_model_class>
train_x = sequence.pad_sequences(train_x, maxlen=maxlen)
 test_x = sequence.pad_sequences(test_x, maxlen=maxlen )
test_sc_tab = []
lr = 9e-3
for i in tqdm(range(FOLDS)) :
dls = get_data(i)
 emb_szs = get_emb_sz(dls)
 model = TabNetModel(emb_szs, len(dls.cont_names), dls.c, n_d=8, n_a=32, n_steps=1);
opt_func = partial(Adam, wd=0.01, eps=1e-5)
 learn = Learner(dls, model, BCEWithLogitsLossFlat() , opt_func=opt_func, lr=lr, model_dir='/kaggle/working/')
 name = 'best_model_tabnet_' + str(i)
 cb = SaveModelCallback(monitor='valid_loss',fname=name ,mode='min')
 lr = 9e-3
learn.fit_one_cycle(30, slice(lr/(2.6**4),lr),cbs=cb)
 learn.load(name)
 test_dl = learn.dls.test_dl(test_features)
 sub = learn.get_preds(dl=test_dl)
 test_sc_tab.append(sub[0].numpy())
 learn.export('/kaggle/working/'+name+'.pkl')
 test_sc_tab = np.array(test_sc_tab )<groupby>
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32' )
avg_prds_tab = test_sc_tab.mean(axis=0 )<feature_engineering>
word_index = tokenizer.word_index
submission_tab = sample_submission.copy()
submission_tab[cols] = avg_prds_tab
submission_tab.loc[submission_tab['sig_id'].isin(test_features.loc[test_features['cp_type'] =='ctl_vehicle', 'sig_id']), train_targets_scored.columns[1:]] = 0
submission_tab['atp-sensitive_potassium_channel_antagonist'] = 0
submission_tab['erbb2_inhibitor'] = 0<save_to_csv>
nb_words = min(max_features, len(word_index))
submission_tab.to_csv('submission_tabnet.csv',index=False )<define_variables>
embedding_matrix = np.zeros(( nb_words, embed_size))
final_prds = np.array(( list(avg_prds),list(avg_prds_tab)) ).mean(axis=0 )<define_variables>
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
fin_wt_prds = avg_prds*(0.75)+ avg_prds_tab*(0.25 )<feature_engineering>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
submission_fin = sample_submission.copy()
submission_fin[cols] = fin_wt_prds
submission_fin.loc[submission_fin['sig_id'].isin(test_features.loc[test_features['cp_type'] =='ctl_vehicle', 'sig_id']), train_targets_scored.columns[1:]] = 0
submission_fin['atp-sensitive_potassium_channel_antagonist'] = 0
submission_fin['erbb2_inhibitor'] = 0<save_to_csv>
def get_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(80, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()
results = submission_fin.copy()
for cl in cols:
results[cl].clip(0.0002, 0.999, inplace = True)
 results.to_csv('submission.csv',index=False )<import_modules>
batch_size = 32
epochs = 2
import numpy as np
import pandas as pd 
<load_from_csv>
X_tra, X_val, y_tra, y_val = train_test_split(train_x, train_y, train_size=0.95, random_state=233 )
df1 = pd.read_csv('.. /input/fork-of-2heads-looper-super-puper/submission.csv')
 df2=pd.read_csv('.. /input/moa-pytorch-feature-engineering-0-01846/submission.csv')
 df3 = pd.read_csv('.. /input/explore-the-magic-of-mean-top-2-of-the-publiclb/submission.csv' )<sort_values>
RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1 )
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<sort_values>
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=2 )
df1 = order_sub(df1)
 df2 = order_sub(df2)
 df3 = order_sub(df3 )<prepare_output>
y_pred = model.predict(test_x, batch_size=1024 )
BLEND=df1.copy()
BLEND.iloc[:,1:] = df1.iloc[:,1:]*0.60 +df2.iloc[:,1:]*0.40<define_variables>
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv' )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL )<load_from_csv>
submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<set_options>
submission.to_csv('submission.csv', index=False )
<load_from_csv><EOS> 
submission.to_csv('submission.csv', index=False )