Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<compute_test_metric>
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<choose_model_class>
start_time = time.time()
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
cores = 4
train1 = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 train = train1
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
max_features = 50000
maxlen = 100
embed_size = 300
def create_model1(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss='binary_crossentropy', 
)
 return model<choose_model_class>
lemmatizer = WordNetLemmatizer()
def penn_to_wn(tag):

 if tag.startswith('J'):
return wn.ADJ
elif tag.startswith('N'):
return wn.NOUN
elif tag.startswith('R'):
return wn.ADV
elif tag.startswith('V'):
return wn.VERB
return None
def clean_text(text):
text = text.replace("<br />", " ")
 return text
def swn_polarity(text):

 tokens_count = 0
sentiment = 0.0
text = clean_text(text)
 tagged_sentence = pos_tag(word_tokenize(text))
for word, tag in tagged_sentence:
wn_tag = penn_to_wn(tag)
 if wn_tag not in(wn.NOUN, wn.ADJ, wn.ADV):
continue
lemma = lemmatizer.lemmatize(word, pos=wn_tag)
 if not lemma:
continue
synsets = wn.synsets(lemma, pos=wn_tag)
 if not synsets:
continue
synset = synsets[0]
swn_synset = swn.senti_synset(synset.name())
 if swn_synset.obj_score() == 1:
sentiment += swn_synset.pos_score() - swn_synset.neg_score()
elif swn_synset.obj_score() != 1:
sentiment += swn_synset.pos_score() - swn_synset.neg_score() + swn_synset.obj_score()
tokens_count += 1
return sentiment
def sentiment(df):
return df.apply(lambda x: swn_polarity(x))
cores = 4
def parallelize_dataframe(df, func):
df_split = np.array_split(df, cores)
 with closing(Pool(cores)) as pool:
df = pd.concat(pool.map(func, df_split))
return df
def sentiment_score(df):
df['senti_score'] = parallelize_dataframe(df['comment_text'], sentiment)
 return df
def create_model2(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1774, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1274, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(764, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(353, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss='binary_crossentropy', 
)
 return model<define_variables>
t1 = time.time()
train = sentiment_score(train)
 t2 = time.time()
print("Time taken is "+str(t2-t1))
print(train.shape )
top_feats = range(1039)
 print(len(top_feats))<normalization>
scaler = preprocessing.MinMaxScaler()
train['senti_score_scaled'] = scaler.fit_transform(k1 )
scaler = StandardScaler()
X = scaler.fit_transform(train.values)
 x_tt = scaler.transform(test_features.values )<data_type_conversions>
train['comment_text'] = train['comment_text'].astype(str )
train = train.astype('float')
 test = test.astype('float' )<init_hyperparams>
tokenizer = text.Tokenizer(num_words=max_features)
 all_text = np.hstack([train['comment_text'].str.lower() ])
 tokenizer.fit_on_texts(all_text)
 print("Fitting Done...Start text to sequence transform")
 train['seq_comment']= tokenizer.texts_to_sequences(train.comment_text.str.lower())
 print("Transform done for train ")
N_STARTS = 2
tf.random.set_seed(42)
 ss_NN = ss_svc.copy()
res_NN = train_targets.copy()
ss_NN.loc[:, train_targets.columns] = 0
res_NN.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=7, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f'Fold {n}')
 if seed%2 == 0:
model = create_model1(len(top_feats))
else :
model = create_model2(len(top_feats))
checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min')
 model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=40, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt], verbose=2 
)
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss_NN.loc[:, train_targets.columns] += test_predict
res_NN.loc[te, train_targets.columns] += val_predict
print('')
 ss_NN.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res_NN.loc[:, train_targets.columns] /= N_STARTS<compute_test_metric>
x_train, x_valid, y_train, y_valid = train_test_split(train, y_train, train_size=0.95, random_state=233)
 print(x_train.shape, x_valid.shape)
 print(y_train.shape, y_valid.shape )
print(f'NN OOF Metric: {log_loss_metric(train_targets, res_NN)}')
 res_NN.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_NN.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )<save_to_csv>
def get_keras_data(dataset):
X = {
'comment_text' : pad_sequences(dataset.seq_comment, maxlen=maxlen),
'senti_score_scaled': np.array(dataset.senti_score)
 }
return X
ss_NN.to_csv('submission.csv', index = False )<import_modules>
X_train = get_keras_data(x_train)
 X_valid = get_keras_data(x_valid )
from fastai.tabular.all import *
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from fastai.callback import *
from tqdm.notebook import tqdm
from ml_stratifiers import MultilabelStratifiedKFold<set_options>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
print(self.interval)
 self.X_val, self.y_val = validation_data
print(self.y_val)
 def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
def get_model1() :
inp = Input(shape = [X_train['comment_text'].shape[1]], name = 'comment_text')
 x = Embedding(max_features, embed_size )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(40, return_sequences=True, dropout = 0.15, recurrent_dropout = 0.15))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation='sigmoid' )(conc)
 model = Model([inp], outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model1 = get_model1()
batch_size = 128
epochs = 1
RocAuc = RocAucEvaluation(validation_data=(X_valid, y_valid), interval=1)
 os.environ['OMP_NUM_THREADS'] = '4'
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train)/batch_size)* epochs
lr_init, lr_fin = 0.009, 0.0045
lr_decay = exp_decay(lr_init, lr_fin, steps)
 K.set_value(model1.optimizer.lr, lr_init)
 K.set_value(model1.optimizer.decay, lr_decay)
 for i in range(3):
hist = model1.fit(X_train, y_train, batch_size=batch_size+(batch_size*(2*i)) , epochs=epochs, validation_data=(X_valid,y_valid), callbacks=[RocAuc], verbose=1)
 print("Model 1 Done" )
pd.set_option('display.max_rows', 50)
 pd.set_option('display.max_columns', 50 )<define_variables>
w2v = gensim.models.KeyedVectors.load_word2vec_format('.. /input/googlenews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)
 print("Done loading model" )
path = Path('.. /input/lish-moa' )<load_from_csv>
vocab_size = len(tokenizer.word_index)+1
EMBEDDING_DIM = 300
embedding_matrix = np.zeros(( vocab_size, EMBEDDING_DIM))
print(embedding_matrix.shape)
 c = 0
c1 = 0
w_Y = []
w_No = []
for word, i in tokenizer.word_index.items() :
if word in w2v:
c +=1
embedding_vector = w2v[word]
w_Y.append(word)
 else:
embedding_vector = None
w_No.append(word)
 c1 +=1
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print(c,c1, len(w_No), len(w_Y))
print(embedding_matrix.shape)
 class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
print(self.interval)
 self.X_val, self.y_val = validation_data
print(self.y_val)
 def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
def get_model2() :
inp = Input(shape = [X_train['comment_text'].shape[1]], name = 'comment_text')
 print("Here")
 x = Embedding(vocab_size, EMBEDDING_DIM, weights = [embedding_matrix], trainable = True )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(40, return_sequences=True, dropout = 0.15, recurrent_dropout = 0.15))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model([inp], outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model2 = get_model2()
RocAuc = RocAucEvaluation(validation_data=(X_valid, y_valid), interval=1)
 os.environ['OMP_NUM_THREADS'] = '4'
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train)/batch_size)* epochs
lr_init, lr_fin = 0.009, 0.0045
lr_decay = exp_decay(lr_init, lr_fin, steps)
 K.set_value(model2.optimizer.lr, lr_init)
 K.set_value(model2.optimizer.decay, lr_decay)
 for i in range(3):
hist = model2.fit(X_train, y_train, batch_size=batch_size+(batch_size*(2*i)) , epochs=epochs, validation_data=(X_valid,y_valid), callbacks=[RocAuc], verbose=1)
 print("Model 2 Done")
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv' )<count_unique_values>
del X_train, X_valid, y_valid
gc.collect()
train_features.sig_id.nunique()<count_values>
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
train_features.cp_type.value_counts()<count_values>
t1 = time.time()
def load_test() :
for df in pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv', chunksize= 150000):
yield df
print("Yield complete")
 test_ids = np.array([], dtype=np.int32)
 preds= np.zeros(( 0,6), dtype = np.int32)
 print("Start Batch Prediction")
 c = 0
for df in load_test() :
c +=1
print("Chunk number is "+str(c))
sentiment_score(df)
 k2 = df['senti_score'].reshape(-1,1)
 df['senti_score_scaled'] = scaler.transform(k2)
 df['seq_comment'] = tokenizer.texts_to_sequences(df.comment_text.str.lower())
 print("Transform done for test ")
 X_test = get_keras_data(df)
 y_pred1 = model1.predict(X_test, batch_size=1024, verbose =1)
 y_pred2 = model2.predict(X_test, batch_size=1024, verbose =1)
 test_id = df['id']
del df['seq_comment'], df['senti_score'], X_test
gc.collect()
k =(y_pred1+y_pred2)/2
print(k.shape)
 preds= np.append(preds,k, axis = 0)
 print(preds.shape)
 test_ids = np.append(test_ids, test_id)
 print("All chunks done")
 t2 = time.time()
print("Total time for Parallel Batch Prediction is "+str(t2-t1))
<count_values><EOS> 
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = preds
submission.to_csv('submission.csv', index=False)
 end_time = time.time()
print("Total time taken is "+str(end_time-start_time))