Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<set_options>
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)
pd.options.display.max_columns = 50<load_from_csv>
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )
best = pd.read_csv(".. /input/mym5files/GMmasmodels_v2.csv")
 best.head()<normalization>
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
data_folder = ".. /input/jigsaw-toxic-comment-classification-challenge/"
pretrained_folder = ".. /input/"
train_filepath = data_folder + "train.csv"
test_filepath = data_folder + "test.csv"
submission_path = data_folder + "submission.csv"
facebook_filepath = pretrained_folder + "fasttext-crawl-300d-2m/crawl-300d-2M.vec"
google_filepath = pretrained_folder + "glove840b300dtxt/glove.840B.300d.txt"
embedding_dim_facebook = 300
embedding_dim_google = 300
c=0.04
best.iloc[:,1:]=best.iloc[:,1:].ewm(com=c,axis=1 ).mean().values<load_from_csv>
hyphens_filepath = ".. /input/cleaning-dictionaries/hyphens_dictionary.bin"
misspellings_filepath = ".. /input/cleaning-dictionaries/misspellings_all_dictionary.bin"
merged_filepath = ".. /input/cleaning-dictionaries/merged_all_dictionary.bin"
hyphens_dict = misspellings_dict = merged_dict = {}
with open(hyphens_filepath, mode='rb')as file: hyphens_dict = pickle.load(file)
 with open(misspellings_filepath, mode='rb')as file: misspellings_dict = pickle.load(file)
 with open(merged_filepath, mode='rb')as file: merged_dict = pickle.load(file)
 print(len(hyphens_dict))
print(len(misspellings_dict))
print(len(merged_dict))
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_validation.csv")
 salesids=sales.id.values
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_evaluation.csv")
 sales.id=salesids
sales.head()<merge>
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}
sub = best.merge(sales[["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]], on = "id")
 sub["_all_"] = "Total"
sub.shape<define_search_space>
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )
qs = np.array([0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995])
 qs.shape
def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios[-1] *= 1.03
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios2(coef=0.15, a=1.2):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.skewnorm.cdf(qs2, a)
 ratios /= ratios[4]
ratios[-1] *= 1.02
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios3(coef=0.15, c=0.5, s=0.1):
qs2 = qs*coef
ratios = stats.powerlognorm.ppf(qs2, c, s)
 ratios /= ratios[4]
ratios[0] *= 0.25
ratios[-1] *= 1.02
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def widen(array, pc):
array[array<1]=array[array<1] *(1 - pc/100)
 array[array>1]=array[array>1] *(1 + pc/100)
 return array
widen_coef=0.5
level_coef_dict = {"id": widen(( get_ratios2(coef=0.3)+get_ratios3(coef=.3, c=0.04, s=0.9)) /2, pc=widen_coef), "item_id": widen(get_ratios2(coef=0.18, a=0.4),pc=widen_coef),
"dept_id": widen(get_ratios(coef=0.04),widen_coef), "cat_id": widen(get_ratios(coef=0.03),widen_coef),
"store_id": widen(get_ratios(coef=0.035),widen_coef), "state_id": widen(get_ratios(coef=0.03),widen_coef),
"_all_": widen(get_ratios(coef=0.025),widen_coef), 
("state_id", "item_id"): widen(get_ratios2(coef=0.21, a=0.75),pc=widen_coef),("state_id", "dept_id"): widen(get_ratios(coef=0.05),widen_coef), 
("store_id","dept_id"): widen(get_ratios(coef=0.07),widen_coef),("state_id", "cat_id"): widen(get_ratios(coef=0.04),widen_coef), 
("store_id","cat_id"): widen(get_ratios(coef=0.055),widen_coef)
 } 
<normalization>
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, hyphens_dict)
 normalized_word = normalize_by_dictionary(normalized_word, merged_dict)
 normalized_word = normalize_by_dictionary(normalized_word, misspellings_dict)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result
a=(( get_ratios2(coef=0.3)+get_ratios3(coef=.3, c=0.04, s=0.9)) /2 ).values
np.where(a==1)[0][0]<feature_engineering>
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )
a[a<1]*0.99,a[a>1]*1.01<statistical_test>
%%time
labels, x_train, x_test = read_data_files(train_filepath, test_filepath )
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_validation', '')}_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_validation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df = pd.concat([df,df] , axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df.loc[df.index >= len(df.index)//2, "id"] = df.loc[df.index >= len(df.index)//2, "id"].str.replace(
 "_validation$", "_evaluation")
 df.shape
df.head() 
<groupby>
%%time
fileObject = open('.. /input/cleaning-dictionaries/tokenizer','rb')
 tokenizer = pickle.load(fileObject)
 x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
salesGitem=sales.groupby('item_id')[['item_id']+[f"d_{i}" for i in range(942, 1942)]].sum()<sort_values>
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )
sales.sort_values('id',inplace=True)
 sales.reset_index(drop=True, inplace = True )<define_search_space>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1)
quantsalesW=[]
for i in range(7):
quantsalesW.append(
 np.expand_dims(
 np.average(
 np.stack(
(( sales.iloc[:,np.arange(-28*13+i,0,7)].quantile(np.array(
 [0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995]), axis=1 ).T ).values.reshape(quantsales.shape[0]*quantsales.shape[1], order='F'), 
(sales.iloc[:,np.arange(-28*3+i,0,7)].quantile(np.array(
 [0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995]), axis=1 ).T ).values.reshape(quantsales.shape[0]*quantsales.shape[1], order='F'))
,-1),axis=-1), -1)
)
 quantsalesW=np.hstack(quantsalesW )<define_variables>
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
medians=np.where(np.array([float(x.split('_')[-2])for x in df.iloc[:274410,0]])== 0.5)[0]
notmedian=np.array([x for x in np.arange(274410)if x not in medians] )<feature_engineering>
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )
<create_dataframe><EOS> 
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )