Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<feature_engineering>
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)
<define_variables>
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )
42000*9*2<split>
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
data_folder = ".. /input/jigsaw-toxic-comment-classification-challenge/"
pretrained_folder = ".. /input/"
train_filepath = data_folder + "train.csv"
test_filepath = data_folder + "test.csv"
submission_path = data_folder + "submission.csv"
facebook_filepath = pretrained_folder + "fasttext-crawl-300d-2m/crawl-300d-2M.vec"
google_filepath = pretrained_folder + "glove840b300dtxt/glove.840B.300d.txt"
embedding_dim_facebook = 300
embedding_dim_google = 300
df.iloc[:int(771120/2),1:].values-df.iloc[int(771120/2):,1:].values<filter>
hyphens_filepath = ".. /input/cleaning-dictionaries/hyphens_dictionary.bin"
misspellings_filepath = ".. /input/cleaning-dictionaries/misspellings_all_dictionary.bin"
merged_filepath = ".. /input/cleaning-dictionaries/merged_all_dictionary.bin"
hyphens_dict = misspellings_dict = merged_dict = {}
with open(hyphens_filepath, mode='rb')as file: hyphens_dict = pickle.load(file)
 with open(misspellings_filepath, mode='rb')as file: misspellings_dict = pickle.load(file)
 with open(merged_filepath, mode='rb')as file: merged_dict = pickle.load(file)
 print(len(hyphens_dict))
print(len(misspellings_dict))
print(len(merged_dict))
np.sum(df.iloc[:int(771120/2),1:].values-df.iloc[int(771120/2):,1:].values )<import_modules>
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc<categorify>
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )
def preprocess_sales(sales, start=1000, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale1",
"sales1","start","sales2","scale2"],
var_name='d', value_name='demand')
 return sales
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar["nb"] = calendar.index + 1
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
def make_dataset(categorize=False ,start=1000, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales_aug.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 calendar = calendar.loc[calendar.nb>=start]
gc.collect()
return sales, calendar 
<compute_test_metric>
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, hyphens_dict)
 normalized_word = normalize_by_dictionary(normalized_word, merged_dict)
 normalized_word = normalize_by_dictionary(normalized_word, misspellings_dict)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result
def qloss_np(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = np.array([qs])
 e = y_true - y_pred
v = np.maximum(q*e,(q-1)*e)
 return np.mean(v )<create_dataframe>
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )
%%time
CATEGORIZE = True;
START = 1000; UPPER = 1970;
maps = {}
mods = {}
sales, calendar = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )<feature_engineering>
%%time
labels, x_train, x_test = read_data_files(train_filepath, test_filepath )
sales["x"] = sales["demand"] / sales["scale1"]<prepare_x_and_y>
%%time
fileObject = open('.. /input/cleaning-dictionaries/tokenizer','rb')
 tokenizer = pickle.load(fileObject)
 x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
NITEMS = 42840
scale = sales['scale1'].values.reshape(( NITEMS, -1))
ids = sales['id'].values.reshape(( NITEMS, -1))
ys = sales[['x','sales1']].values.reshape(( NITEMS, -1, 2))
Z = sales[FEATS].values.reshape(( NITEMS, -1, len(FEATS)) )<define_variables>
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )
LEN = 1969 - START + 1
MAX_LAG = max(LAGS)
 print(LEN, MAX_LAG )<define_variables>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1)
CATCOLS = ['snap_CA', 'snap_TX', 'snap_WI', 'wday', 'month', 'year', 'event_name', 'nday',
'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']<import_modules>
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x1 = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences = True))(x1)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 y = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences = True))(x1)
 y = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(y)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 avg_pool2 = GlobalAveragePooling1D()(y)
 max_pool2 = GlobalMaxPooling1D()(y)
 x = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
 x = Dense(6, activation = "sigmoid" )(x)
 model = Model(inputs = words, outputs = x)
 model.compile(loss = "binary_crossentropy", optimizer = "adam", metrics = ["accuracy"])
 return model
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
import tensorflow as tf
import itertools as it<categorify>
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )
<prepare_x_and_y><EOS> 
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )