Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<compute_test_metric>
import sys, os, re, csv, codecs, numpy as np, pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, GRU, Embedding, Dropout, Activation
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from nltk.tokenize import TweetTokenizer
from unidecode import unidecode
import gc
from keras import backend as K
meaneduc_correction(test_meaneduc_null, test, h_ids )<count_missing_values>
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)
test_null_counts = test.isnull().sum()
test_null_counts[test_null_counts>0]<count_unique_values>
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE1=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'
EMBEDDING_FILE2=f'{path}fasttext-crawl-300d-2m/rawl-300d-2M.vec'
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
train['Id'].unique().size, test['Id'].unique().size<define_variables>
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )
cols = ['v2a1', 'hacdor', 'rooms', 'hacapo', 'v14a', 'refrig', 'v18q1',
'r4h3', 'r4m3', 'r4t3', 'tamhog', 'tamviv', 'hhsize', 'paredblolad',
'paredzocalo', 'paredpreb', 'pareddes', 'paredmad', 'paredzinc',
'paredfibras', 'paredother', 'pisomoscer', 'pisocemento',
'pisoother', 'pisonatur', 'pisonotiene', 'pisomadera', 'techozinc',
'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano', 'public',
'planpri', 'noelec', 'coopele', 'sanitario1', 'sanitario2',
'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5',
'elimbasu6', 'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2',
'etecho3', 'eviv1', 'eviv2', 'eviv3', 'hogar_nin', 'hogar_adul',
'hogar_mayor', 'hogar_total', 'dependency', 'meaneduc', 'bedrooms',
'overcrowding', 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4',
'tipovivi5', 'computer', 'television', 'qmobilephone', 'lugar1',
'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'Target']<categorify>
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 def check_for_wrong_data(data, columns, labelE, gpby='idhogar'):

 id_head_zero = []
idhogarId_f = []
cols_f = []
mem_f = []
grouped = data.groupby(gpby, sort=True)
 for gid in range(len(grouped)) :
members = grouped.get_group(gid)
 h_Head = members.loc[(members['parentesco1'] == 1)]
if h_Head.shape[0] == 0:
id_head_zero.append(members['idhogar'].values[0])
 continue
idhogarId_w = []
cols_t = []
mem_t = []
if members.shape[0] > 1:
for col in columns:
for m in members.iterrows() :
if h_Head[col].values[0] != m[1][col]:
if h_Head['idhogar'].values[0] not in idhogarId_w : idhogarId_w.append(h_Head['idhogar'].values[0])
 if col not in cols_t : cols_t.append(col)
 if m[1]['Id'] not in mem_t : mem_t.append(m[1]['Id'])
 data.loc[(train['Id'] == m[1]['Id']), col] = h_Head[col].values[0]
idhogarId_f.append(idhogarId_w); cols_f.append(cols_t); mem_f.append(mem_t)
 if len(idhogarId_w)> 0:
for i in range(len(idhogarId_w)) :
print("Household with Id: "
+str(labelE.inverse_transform([idhogarId_w[i]])[0])
 +" has " + str(len(mem_t)) + " member(s)with diff.value(s)of " + str(len(cols_t)) + " column(s )."
+ " " + str(cols_t))
return id_head_zero, idhogarId_f, cols_f, mem_f<define_variables>
embedding_dim_facebook = 300
embedding_dim_google = 300
max_features = 20000
maxlen = 100
id_head_zero, *_ = check_for_wrong_data(train, cols, lb2 )<filter>
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}
train.loc[(train['idhogar'] == id_head_zero[11])]<compute_test_metric>
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )
id_head_zero, *_ = check_for_wrong_data(test, cols, lb4 )<define_variables>
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result
possible_outliers = [columns[0]] + [columns[98]]; columns[0], columns[98]<filter>
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )
train.loc[(train['v2a1'] > 300000), ['idhogar', 'v2a1', 'Target']].query("Target != 4" )<filter>
%%time
labels, x_train, x_test = read_data_files(TRAIN_DATA_FILE, TEST_DATA_FILE )
train.loc[(train['v2a1'] > 2000000), ['idhogar', 'v2a1', 'Target']]<filter>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
train.loc[(train['meaneduc'] > 25), ['idhogar', 'Target', 'meaneduc']].query("Target != 4" )<define_variables>
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )
columns = ['v2a1', 'rooms', 'tamhog', 'overcrowding', 'v18q1', 'r4t3', 'meaneduc', 'qmobilephone', 'Target']<drop_column>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1 )
train.drop(['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding',
'SQBdependency', 'SQBmeaned', 'agesq'], axis = 1, inplace=True )<drop_column>
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
test.drop(['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding',
'SQBdependency', 'SQBmeaned', 'agesq'], axis = 1, inplace=True )<feature_engineering>
%%time
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )
DropCols = ['energcocinar1', 'energcocinar4', 'energcocinar3', 'energcocinar2', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']
train['CookingType'] = np.argmax(np.array(train[[ 'energcocinar1', 'energcocinar4', 'energcocinar3', 'energcocinar2' ]]), axis=1)
 train['WallType'] = np.argmax(np.array(train[[ 'epared1', 'epared2', 'epared3' ]]), axis=1)
 train['RoofType'] = np.argmax(np.array(train[[ 'etecho1', 'etecho2', 'etecho3' ]]), axis=1)
 train['FloorType'] = np.argmax(np.array(train[[ 'eviv1', 'eviv2', 'eviv3' ]]), axis=1)
 train['HouseType'] = np.argmax(np.array(train[[ 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5' ]]), axis=1)
 test['CookingType'] = np.argmax(np.array(test[[ 'energcocinar1', 'energcocinar4', 'energcocinar3', 'energcocinar2' ]]), axis=1)
 test['WallType'] = np.argmax(np.array(test[[ 'epared1', 'epared2', 'epared3' ]]), axis=1)
 test['RoofType'] = np.argmax(np.array(test[[ 'etecho1', 'etecho2', 'etecho3' ]]), axis=1)
 test['FloorType'] = np.argmax(np.array(test[[ 'eviv1', 'eviv2', 'eviv3' ]]), axis=1)
 test['HouseType'] = np.argmax(np.array(test[[ 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5' ]]), axis=1)
 train.drop(DropCols, axis=1, inplace=True)
 test.drop(DropCols, axis=1, inplace=True)
 test.shape, train.shape<feature_engineering>
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )