Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<train_model>
print(os.listdir(".. /input"))
X = sales[COLS].values
sw = sales['sales1'].values
sc = sales['scale1'].values
Z = calendar[FE].values
Xs = X / sc[:, np.newaxis]
print(X.shape, Z.shape )<train_model>
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 MAX_SEQUENCE_LENGTH = 150
MAX_NB_WORDS = 100000
EMBEDDING_DIM = 300
VALIDATION_SPLIT = 0.1
num_lstm = 300
num_dense = 256
rate_drop_lstm = 0.2
rate_drop_dense = 0.2
act = 'relu'
%%time
step1 = MultipleRegression(reg=0.001)
 step1.fit(Xs, Z[:-28] )<predict_on_test>
color = sns.color_palette()
sns.set_style("dark" )
Xp = step1.predict(Z )<compute_test_metric>
def cleanData(text, stemming = False, lemmatize=False):
text = text.lower().split()
text = " ".join(text)
 text = re.sub(r"[^A-Za-z0-9^,!.\/'+\-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 if stemming:
st = PorterStemmer()
txt = " ".join([st.stem(w)for w in text.split() ])
 if lemmatize:
wordnet_lemmatizer = WordNetLemmatizer()
txt = " ".join([wordnet_lemmatizer.lemmatize(w)for w in text.split() ])
 return text
print("mae:", mean_absolute_error(Xs[:, -28:], Xp[:,-56:-28]))
print("mse:", mean_squared_error(Xs[:, -28:], Xp[:,-56:-28], squared=False))<define_variables>
print('Indexing word vectors')
 count = 0
embeddings_index = {}
f = open(EMBEDDING_FILE)
 for line in f:
values = line.split()
word = ' '.join(values[:-300])
 coefs = np.asarray(values[-300:], dtype='float32')
 embeddings_index[word] = coefs.reshape(-1)
 coef = embeddings_index[word]
f.close()
print('Found %d word vectors of glove.' % len(embeddings_index))
emb_mean,emb_std = coef.mean() , coef.std()
print(emb_mean,emb_std)
 print('Total %s word vectors.' % len(embeddings_index))
resid = Xs - Xp[:, :-28]<define_search_space>
import os
import re
import csv
import codecs
import numpy as np
import pandas as pd
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from string import punctuation
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import *
from keras.layers import concatenate, CuDNNGRU
from keras.models import Model
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping, ModelCheckpoint
import sys
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
cp = np.quantile(resid, qs, axis=1 ).T<filter>
print('Processing text dataset')
 train_df['comment_text'] = train_df['comment_text'].map(lambda x: cleanData(x, stemming=False, lemmatize=False))
test_df['comment_text'] = test_df['comment_text'].map(lambda x: cleanData(x, stemming=False, lemmatize=False))
special_character_removal=re.compile(r'[^a-z\d ]',re.IGNORECASE)
 replace_numbers=re.compile(r'\d+',re.IGNORECASE)
 def text_to_wordlist(text, remove_stopwords=False, stem_words=False):
text = text.lower().split()
if remove_stopwords:
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops]
text = " ".join(text)
 text=special_character_removal.sub('',text)
 text=replace_numbers.sub('n',text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return(text)
 list_sentences_train = train_df["comment_text"].fillna("NA" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train_df[list_classes].values
list_sentences_test = test_df["comment_text"].fillna("NA" ).values
comments = []
for text in list_sentences_train:
comments.append(text_to_wordlist(text))
test_comments=[]
for text in list_sentences_test:
test_comments.append(text_to_wordlist(text))
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
 tokenizer.fit_on_texts(comments + test_comments)
 sequences = tokenizer.texts_to_sequences(comments)
 test_sequences = tokenizer.texts_to_sequences(test_comments)
 word_index = tokenizer.word_index
print('Found %s unique tokens' % len(word_index))
data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
 print('Shape of data tensor:', data.shape)
 print('Shape of label tensor:', y.shape)
 test_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)
 print('Shape of test_data tensor:', test_data.shape )
cp = cp[:,np.newaxis, :]<prepare_x_and_y>
data_post = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH,padding='post', truncating='post')
 print('Shape of data tensor:', data_post.shape)
 print('Shape of label tensor:', y.shape)
 test_data_post = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')
 print('Shape of test_data tensor:', test_data_post.shape )
Zq = Xp[:,:,np.newaxis] + cp<feature_engineering>
print('Preparing embedding matrix')
 nb_words = min(MAX_NB_WORDS, len(word_index))
embedding_matrix = np.zeros(( nb_words, EMBEDDING_DIM))
for word, i in word_index.items() :
if i >= MAX_NB_WORDS:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1)== 0))
Zq = Zq.clip(0 )<split>
max_features=100000
maxlen=150
embed_size=300
pv = Zq[:,-56:-28, :]
pe = Zq[:,-28:, :]<normalization>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
sc= sc[:, np.newaxis]<create_dataframe>
gru_len = 128
Routings = 5
Num_capsule = 10
Dim_capsule = 16
dropout_p = 0.25
rate_drop_dense = 0.28
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule=10, dim_capsule=16, routings=5, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 ")
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 ")
 ")
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
")
 u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
")
 b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
")
 ")
 ")
 if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
names = [f"F{i+1}" for i in range(28)]
piv = pd.DataFrame(ids, columns=["id"] )<feature_engineering>
def get_model() :
input1_pre = Input(shape=(maxlen,))
embed_layer1_pre = Embedding(max_features,
embed_size,
input_length=maxlen,
weights=[embedding_matrix],
trainable=False )(input1_pre)
 embed_layer1_pre = SpatialDropout1D(0.4 )(embed_layer1_pre)
 x_pre = Bidirectional(CuDNNGRU(128, return_sequences=True))(embed_layer1_pre)
 capsule_pre = Capsule(num_capsule=10, dim_capsule=16, routings=5,share_weights=True )(x_pre)
 capsule_pre = GlobalMaxPooling1D()(capsule_pre)
 capsule_pre = Dropout(0.25 )(capsule_pre)
 input1_post = Input(shape=(maxlen,))
embed_layer1_post = Embedding(max_features,
embed_size,
input_length=maxlen,
weights=[embedding_matrix],
trainable=False )(input1_post)
 embed_layer1_post = SpatialDropout1D(0.4 )(embed_layer1_post)
 x_post = Bidirectional(CuDNNGRU(128, return_sequences=True))(embed_layer1_post)
 capsule_post = Capsule(num_capsule=10, dim_capsule=16, routings=5,share_weights=True )(x_post)
 capsule_post = GlobalMaxPooling1D()(capsule_post)
 capsule_post = Dropout(0.25 )(capsule_post)
 concat = concatenate([capsule_pre,capsule_post])
 output = Dense(6, activation='sigmoid' )(concat)
 model = Model(inputs=[input1_pre,input1_post], outputs=output)
 model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=1e-3,decay=0),
metrics=['accuracy'])
 return model
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sc, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*sc, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
<create_dataframe>
file_path = "capsule_val0.05.h5"
model = get_model()
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=3)
 callbacks_list = [checkpoint, early]
hist = model.fit([data, data_post], y, epochs=10, batch_size=128, shuffle=True, validation_split=0.05, callbacks = callbacks_list,
verbose=1)
 best_score = min(hist.history['val_loss'])
 print(best_score )
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2<save_to_csv>
test_predicts_list = []
def train_folds(data,data_post, y,fold_count=10):
print("Starting to train models...")
 fold_size = len(data)// fold_count
models = []
for fold_id in range(0, fold_count):
fold_start = fold_size * fold_id
fold_end = fold_start + fold_size
if fold_id == fold_size - 1:
fold_end = len(data)
 print("Fold {0}".format(fold_id))
train_x = np.concatenate([data[:fold_start], data[fold_end:]])
 train_xp = np.concatenate([data_post[:fold_start], data_post[fold_end:]])
 train_y = np.concatenate([y[:fold_start], y[fold_end:]])
 val_x = data[fold_start:fold_end]
val_xp = data_post[fold_start:fold_end]
val_y = y[fold_start:fold_end]
file_path="capsule_fold{0}.h5".format(fold_id)
 model = get_model()
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=3)
 RocAuc = RocAucEvaluation(validation_data=([val_x, val_xp], val_y), interval=1)
 callbacks_list = [checkpoint, early,RocAuc]
hist = model.fit([train_x, train_xp], train_y, epochs=15, batch_size=128, shuffle=True,
validation_data=([val_x, val_xp], val_y), callbacks = callbacks_list, verbose=1)
 model.load_weights(file_path)
 best_score = min(hist.history['val_loss'])
 print("Fold {0} loss {1}".format(fold_id, best_score))
print("Predicting validation...")
 val_predicts_path = "capsule_val_predicts{0}.npy".format(fold_id)
 val_predicts = model.predict([val_x, val_xp], batch_size=1024, verbose=1)
 np.save(val_predicts_path, val_predicts)
 print("Predicting results...")
 test_predicts_path = "capsule_test_predicts{0}.npy".format(fold_id)
 test_predicts = model.predict([test_data, test_data_post], batch_size=1024, verbose=1)
 test_predicts_list.append(test_predicts)
 np.save(test_predicts_path, test_predicts )
sub.to_csv("submission.csv", index=False )<import_modules>
train_folds(data, data_post, y )
<set_options><EOS> 
CLASSES = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
test_predicts_am = np.zeros(test_predicts_list[0].shape)
 for fold_predict in test_predicts_list:
test_predicts_am += fold_predict
test_predicts_am =(test_predicts_am / len(test_predicts_list))
test_ids = test_df["id"].values
test_ids = test_ids.reshape(( len(test_ids), 1))
test_predicts_am = pd.DataFrame(data=test_predicts_am, columns=CLASSES)
 test_predicts_am["id"] = test_ids
test_predicts_am = test_predicts_am[["id"] + CLASSES]
test_predicts_am.to_csv("10fold_capsule_am.csv", index=False )