Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<feature_engineering>
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/test.csv")
 embedding_path = ".. /input/glove840b300dtxt/glove.840B.300d.txt"
embed_size = 300
max_features = 100000
max_len = 150
test['adult_num'] = test['hogar_adul'] - test['hogar_mayor']
test['adult_rate'] = test['adult_num'] / test['hogar_total']
test['dependency_num'] = test['hogar_nin'] + test['hogar_mayor']
test['dependency_rate'] = test['dependency_num'] / test['hogar_total']
test['adult_dependency_rate'] = test['adult_num'] /(test['dependency_num']+0.1)
 test['children_rate'] = test['hogar_nin'] / test['hogar_total']
test['elder_rate'] = test['hogar_mayor'] / test['hogar_total']
test['rent_per_person'] = test['v2a1'] / test['hogar_total']
test['rent_per_adult'] = test['v2a1'] /(test['adult_num']+0.1)
 test['head_is_adult'] =(test['adult_num'] > 0 ).astype(int)
 test['bedroom_per_person'] = test['bedrooms'] / test['hogar_total']
test['bedroom_per_adult'] = test['bedrooms'] /(test['adult_num']+0.1)
 test['rent_per_room'] = test['v2a1'] / test['rooms']<prepare_x_and_y>
repl = {
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " frown ",
":(": " frown ",
":s": " frown ",
":-s": " frown ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
del trate, tete
train.drop(["new_comment_text"], axis=1, inplace=True)
 test.drop(["new_comment_text"], axis=1, inplace=True )
X = df.drop(['Id', 'idhogar', 'Target'], axis=1 ).fillna(0)
 y = df.Target
test_X = test.drop(['Id', 'idhogar'], axis=1 ).fillna(0 )<normalization>
repl = {
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " frown ",
":(": " frown ",
":s": " frown ",
":-s": " frown ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
del trate, tete
train.drop(["new_comment_text"], axis=1, inplace=True)
 test.drop(["new_comment_text"], axis=1, inplace=True)
 train_text = train['comment_text']
test_text = test['comment_text']
X_ss = pd.DataFrame(StandardScaler().fit(X ).transform(X), columns=X.columns)
 mds = manifold.MDS(2, max_iter=100, n_init=1);
trans_data = mds.fit_transform(X_ss )<feature_engineering>
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
 trans_data = tsne.fit_transform(X_ss)
<split>
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return re_tok.sub(r' \1 ', s ).split()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1, test_size=0.2 )<find_best_params>
cont_patterns = [ 
(b'US', b'United States'), 
(b'IT', b'Information Technology'), 
(b'(W|w)on't', b'will not'), 
(b'(C|c)an't', b'can not'), 
(b'(I|i)'m', b'i am'), 
(b'(A|a)in't', b'is not'), 
(b'(\w+)'ll', b'\g<1> will'), 
(b'(\w+)n't', b'\g<1> not'), 
(b'(\w+)'ve', b'\g<1> have'), 
(b'(\w+)'s', b'\g<1> is'), 
(b'(\w+)'re', b'\g<1> are'), 
(b'(\w+)'d', b'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def prepare_for_char_n_gram(text):

 in r""
clean = bytes(text.lower() , encoding="utf-8")
 and \t
clean = clean.replace(b"
", b" ")
 clean = clean.replace(b"\t", b" ")
 clean = clean.replace(b"\b", b" ")
 clean = clean.replace(b"\r", b" ")
 for(pattern, repl)in patterns:
clean = re.sub(pattern, repl, clean)
 exclude = re.compile(b'[%s]' % re.escape(bytes(string.punctuation, encoding='utf-8')))
 clean = b" ".join([exclude.sub(b'', token)for token in clean.split() ])
 clean = re.sub(b"\d+", b" ", clean)
 clean = re.sub(b'\s+', b' ', clean)
 clean = re.sub(b'\s+$', b'', clean)
 clean = re.sub(b" ", b" ", clean)
 clean = b"" + clean + b""
return str(clean, 'utf-8')
 def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))
scores_knn_distance = pd.DataFrame(columns=['neighbours','score'])
 for i_ in range(3,20):
clf_knn = neighbors.KNeighborsClassifier(n_neighbors=i_, weights="distance")
 model_knn = clf_knn.fit(X_train,y_train)
 score = f1_score(y_test, model_knn.predict(X_test), average = 'macro')
 scores_knn_distance = scores_knn_distance.append({'neighbours':i_,'score':score},ignore_index=True)
 print('distance, neighbours=',i_,', f1_score=',score.mean() )<find_best_params>
def get_indicators_and_clean_comments(df):

 ", x))
df["raw_word_len"] = df["comment_text"].apply(lambda x: len(x.split()))
df["raw_char_len"] = df["comment_text"].apply(lambda x: len(x))
df["nb_upper"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[A-Z]", x))
df["nb_fk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ff]\S{2}[Kk]", x))
df["nb_sk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ss]\S{2}[Kk]", x))
df["nb_dk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[dD]ick", x))
df["nb_you"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\W[Yy]ou\W", x))
df["nb_mother"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wmother\W", x))
df["nb_ng"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wnigger\W", x))
df["start_with_columns"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"^\:+", x))
df["has_timestamp"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\d{2}|:\d{2}", x))
df["has_date_long"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{2}:\d{2}, \d{1,2} \w+ \d{4}", x))
df["has_date_short"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{1,2} \w+ \d{4}", x))
df["has_http"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"http[s]{0,1}://\S+", x))
df["has_mail"] = df["comment_text"].apply(
 lambda x: count_regexp_occ(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', x)
)
 df["has_emphasize_equal"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\={2}.+\={2}", x))
df["has_emphasize_quotes"] = df["comment_text"].apply(lambda x: count_regexp_occ(r""{4}\S+"{4}", x))
df["clean_comment"] = df["comment_text"].apply(lambda x: prepare_for_char_n_gram(x))
df["clean_word_len"] = df["clean_comment"].apply(lambda x: len(x.split()))
df["clean_char_len"] = df["clean_comment"].apply(lambda x: len(x))
df["clean_chars"] = df["clean_comment"].apply(lambda x: len(set(x)))
 df["clean_chars_ratio"] = df["clean_comment"].apply(lambda x: len(set(x)))/ df["clean_comment"].apply(
 lambda x: 1 + min(99, len(x)) )
scores_knn_uniform = pd.DataFrame(columns=['neighbours','score'])
 for i_ in range(3,20):
clf_knn = neighbors.KNeighborsClassifier(n_neighbors=i_, weights="uniform")
 model_knn = clf_knn.fit(X_train,y_train)
 score = f1_score(y_test, model_knn.predict(X_test), average = 'macro')
 scores_knn_uniform = scores_knn_uniform.append({'neighbours':i_,'score':score},ignore_index=True)
 print('uniform, neighbours=',i_,', f1_score=',score.mean() )<train_model>
train_text = train['clean_comment'].fillna("")
 test_text = test['clean_comment'].fillna("" )
clf_svm = svm.SVC(gamma='auto')
 model_svm = clf_svm.fit(X_train, y_train )<compute_train_metric>
np.random.seed(32)
 os.environ["OMP_NUM_THREADS"] = "4"
scores_svm = f1_score(y_test, model_svm.predict(X_test), average = 'macro')
 print('svm, f1_score=',scores_svm.mean() )<train_model>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
clf_sgd = SGDClassifier(loss="hinge", penalty="l1",
alpha=0.01, max_iter=200, fit_intercept=True)
 model_sgd = clf_sgd.fit(X,y )<compute_train_metric>
y = train[class_names].values
X_train, X_valid, Y_train, Y_valid = train_test_split(train_text, y, test_size = 0.05 )
scores_sgd = f1_score(y_test, model_sgd.predict(X_test), average = 'macro')
 print('sgd, f1_score=',scores_sgd.mean() )<train_model>
raw_text_train = X_train
raw_text_valid = X_valid
raw_text_test = test_text
tk = Tokenizer(num_words = max_features, lower = True)
 tk.fit_on_texts(raw_text_train)
 X_train["comment_seq"] = tk.texts_to_sequences(raw_text_train)
 X_valid["comment_seq"] = tk.texts_to_sequences(raw_text_valid)
 test["comment_seq"] = tk.texts_to_sequences(raw_text_test)
 X_train = pad_sequences(X_train.comment_seq, maxlen = max_len)
 X_valid = pad_sequences(X_valid.comment_seq, maxlen = max_len)
 test = pad_sequences(test.comment_seq, maxlen = max_len )
clf_mlp = MLPClassifier(hidden_layer_sizes=(12,), random_state=1, max_iter=1000, warm_start=True)
 model_mlp = clf_mlp.fit(X_train, y_train )<compute_test_metric>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in open(embedding_path))
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
scores_mlp = f1_score(y_test, model_mlp.predict(X_test), average = 'macro')
 print('mlp f1_score=',scores_mlp.mean() )<train_model>
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 ra_val = RocAucEvaluation(validation_data=(X_valid, Y_valid), interval = 1)
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 5)
 def build_model(lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
inp = Input(shape =(max_len,))
x = Embedding(max_features, embed_size, weights = [embedding_matrix], trainable = False )(inp)
 x = SpatialDropout1D(dr )(x)
 x = Bidirectional(GRU(units, return_sequences = True))(x)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 x = Dense(6, activation = "sigmoid" )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
 model.summary()
history = model.fit(X_train, Y_train, batch_size = 128, epochs = 2, validation_data =(X_valid, Y_valid),
verbose = 1, callbacks = [ra_val, check_point, early_stop])
 model = load_model(file_path)
 return model
clf_ada = AdaBoostClassifier(DecisionTreeClassifier(max_depth=5, min_samples_split=30, min_samples_leaf=30),
algorithm="SAMME.R",n_estimators=500, learning_rate=1)
 model_ada = clf_ada.fit(X_train,y_train )<compute_test_metric>
model = build_model(lr = 1e-3, lr_d = 0, units = 128, dr = 0.2 )
<train_model><EOS> 
pred = model.predict(test, batch_size = 1024, verbose = 1)
 submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 submission[class_names] =(pred)
 submission.to_csv("submission.csv", index = False)
 print(submission.values[0] )