Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.initializers import he_normal, he_uniform, glorot_normal, glorot_uniform
from keras import backend as K
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.models import Model
from keras.layers import Dense, Embedding, Bidirectional, CuDNNGRU, GlobalAveragePooling1D
from keras.layers import CuDNNLSTM, GlobalMaxPooling1D, concatenate, Input, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import f1_score
import matplotlib.pyplot as plt
from tqdm import tqdm
from statistics import mean, median, stdev
from numpy import amax, amin
import gc
import time
import pandas as pd
import numpy as np
import tensorflow as tf
import operator
import re
import os
import math
def handle_swears(text):
text = re.sub(replace_with_fuck, ' fuck ', text)
 return text
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def get_coefs(word, *arr):
return word, np.asarray(arr,dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<set_options>
batch_size = 1024
epochs = 18
current_embd = "Glove"
question_length = 100
max_eval_size = 15000
DATA_SPLIT_SEED = 2018
K_FOLDS = 5
K_FOLD_EPOCHS = int(epochs/K_FOLDS)
 seed_nb=14
np.random.seed(seed_nb)
 tf.set_random_seed(seed_nb )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim 
<choose_model_class>
paraules_prohibides = ['2g1c', '2 girls 1 cup', 'acrotomophilia', 'alabama hot pocket', 'alaskan pipeline', 'anal', 'anilingus', 'anus',
'apeshit', 'arsehole', 'ass', 'asshole', 'assmunch', 'auto erotic', 'autoerotic', 'babeland', 'baby batter',
'baby juice', 'ball gag', 'ball gravy', 'ball kicking', 'ball licking', 'ball sack', 'ball sucking', 'bangbros',
'bareback', 'barely legal', 'barenaked', 'bastard', 'bastardo', 'bastinado', 'bbw', 'bdsm', 'beaner', 'beaners',
'beaver cleaver', 'beaver lips', 'bestiality', 'big black', 'big breasts', 'big knockers', 'big tits', 'bimbos',
'birdlock', 'bitch', 'bitches', 'black cock', 'blonde action', 'blonde on blonde action', 'blowjob', 'blow job',
'blow your load', 'blue waffle', 'blumpkin', 'bollocks', 'bondage', 'boner', 'boob', 'boobs', 'booty call',
'brown showers', 'brunette action', 'bukkake', 'bulldyke', 'bullet vibe', 'bullshit', 'bung hole', 'bunghole',
'busty', 'butt', 'buttcheeks', 'butthole', 'camel toe', 'camgirl', 'camslut', 'camwhore', 'carpet muncher',
'carpetmuncher', 'chocolate rosebuds', 'circlejerk', 'cleveland steamer', 'clit', 'clitoris', 'clover clamps',
'clusterfuck', 'cock', 'cocks', 'coprolagnia', 'coprophilia', 'cornhole', 'coon', 'coons', 'creampie', 'cum',
'cumming', 'cunnilingus', 'cunt', 'darkie', 'date rape', 'daterape', 'deep throat', 'deepthroat', 'dendrophilia',
'dick', 'dildo', 'dingleberry', 'dingleberries', 'dirty pillows', 'dirty sanchez', 'doggie style', 'doggiestyle',
'doggy style', 'doggystyle', 'dog style', 'dolcett', 'domination', 'dominatrix', 'dommes', 'donkey punch',
'double dong', 'double penetration', 'dp action', 'dry hump', 'dvda', 'eat my ass', 'ecchi', 'ejaculation',
'erotic', 'erotism', 'escort', 'eunuch', 'faggot', 'fecal', 'felch', 'fellatio', 'feltch', 'female squirting',
'femdom', 'figging', 'fingerbang', 'fingering', 'fisting', 'foot fetish', 'footjob', 'frotting', 'fuck',
'fuck buttons', 'fuckin', 'fucking', 'fucktards', 'fudge packer', 'fudgepacker', 'futanari', 'gang bang',
'gay sex', 'genitals', 'giant cock', 'girl on', 'girl on top', 'girls gone wild', 'goatcx', 'goatse', 'god damn',
'gokkun', 'golden shower', 'goodpoop', 'goo girl', 'goregasm', 'grope', 'group sex', 'g-spot', 'guro', 'hand job',
'handjob', 'hard core', 'hardcore', 'hentai', 'homoerotic', 'honkey', 'hooker', 'hot carl', 'hot chick', 'how to kill',
'how to murder', 'huge fat', 'humping', 'incest', 'intercourse', 'jack off', 'jail bait', 'jailbait', 'jelly donut',
'jerk off', 'jigaboo', 'jiggaboo', 'jiggerboo', 'jizz', 'juggs', 'kike', 'kinbaku', 'kinkster', 'kinky', 'knobbing',
'leather restraint', 'leather straight jacket', 'lemon party', 'lolita', 'lovemaking', 'make me come', 'male squirting',
'masturbate', 'menage a trois', 'milf', 'missionary position', 'motherfucker', 'mound of venus', 'mr hands', 'muff diver',
'muffdiving', 'nambla', 'nawashi', 'negro', 'neonazi', 'nigga', 'nigger', 'nig nog', 'nimphomania', 'nipple', 'nipples',
'nsfw images', 'nude', 'nudity', 'nympho', 'nymphomania', 'octopussy', 'omorashi', 'one cup two girls', 'one guy one jar',
'orgasm', 'orgy', 'paedophile', 'paki', 'panties', 'panty', 'pedobear', 'pedophile', 'pegging', 'penis', 'phone sex',
'piece of shit', 'pissing', 'piss pig', 'pisspig', 'playboy', 'pleasure chest', 'pole smoker', 'ponyplay', 'poof',
'poon', 'poontang', 'punany', 'poop chute', 'poopchute', 'porn', 'porno', 'pornography', 'prince albert piercing',
'pthc', 'pubes', 'pussy', 'queaf', 'queef', 'quim', 'raghead', 'raging boner', 'rape', 'raping', 'rapist', 'rectum',
'reverse cowgirl', 'rimjob', 'rimming', 'rosy palm', 'rosy palm and her 5 sisters', 'rusty trombone', 'sadism',
'santorum', 'scat', 'schlong', 'scissoring', 'semen', 'sexo', 'sexy', 'shaved beaver', 'shaved pussy',
'shemale', 'shibari', 'shit', 'shitblimp', 'shitty', 'shota', 'shrimping', 'skeet', 'slanteye', 'slut', 's&m',
'smut', 'snatch', 'snowballing', 'sodomize', 'sodomy', 'spic', 'splooge', 'splooge moose', 'spooge', 'spread legs',
'spunk', 'strap on', 'strapon', 'strappado', 'strip club', 'style doggy', 'suck', 'sucks', 'suicide girls', 'sultry women',
'swastika', 'swinger', 'tainted love', 'taste my', 'tea bagging', 'threesome', 'throating', 'tied up', 'tight white',
'tit', 'tits', 'titties', 'titty', 'tongue in a', 'topless', 'tosser', 'towelhead', 'tranny', 'tribadism', 'tub girl',
'tubgirl', 'tushy', 'twat', 'twink', 'twinkie', 'two girls one cup', 'undressing', 'upskirt', 'urethra play', 'urophilia',
'vagina', 'venus mound', 'vibrator', 'violet wand', 'vorarephilia', 'voyeur', 'vulva', 'wank', 'wetback', 'wet dream',
'white power', 'wrapping men', 'wrinkled starfish', 'xx', 'xxx', 'yaoi', 'yellow showers', 'yiffy', 'zoophilia', '🖕']
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights = [embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x)
 ])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result =Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs =words, outputs =[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<load_from_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 test['target']=-1
df = pd.concat([train ,test] )
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<feature_engineering>
del train, test; gc.collect() ; time.sleep(5 )
train_df['comment_text'] = train_df['comment_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['comment_text'] = test_df['comment_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
gc.collect()
gc.collect()<prepare_x_and_y>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in tqdm(open(file)) if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in tqdm(open(file, encoding='latin')))
 return embeddings_index
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )<feature_engineering>
embed_glove = load_embed('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt' )
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >=0.5, True, False )<string_transform>
embed_paragram = load_embed('.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt' )
tokenizer = text.Tokenizer(filters = CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train,maxlen=MAX_LEN)
 x_test= sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
my_embedding_matrix = dict()
for k1,v1 in embed_glove.items() :
my_val = v1
if k1 in embed_paragram.keys() :
my_val =(v1 + embed_paragram[k1])/2
my_embedding_matrix[k1] = my_val
for k1,v1 in embed_paragram.items() :
if k1 not in embed_glove.keys() :
my_embedding_matrix[k1] = v1
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN]*(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])*train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights/=sample_weights.mean()
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index,f)for f in EMBEDDING_FILES], axis =-1)
 del train_df, tokenizer
gc.collect()
checkpoint_predictions = []
weights = [] 
<train_model>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks = [
LearningRateScheduler(lambda _: 1e-3*(0.55**global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 del model
gc.collect()
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<import_modules>
df['size'] = df['question_text'].str.len()
print(mean(df['size']))
print(median(df['size']))
print(stdev(df['size']))
print(amax(df['size']))
print(amin(df['size']))
df = df.drop(['size'], axis=1 )
import numpy as np
import pandas as pd
import gc
import re
from keras.preprocessing import text, sequence
import keras.backend as K
from keras.models import Model
from keras.layers import Input, Dense, Dropout, Embedding, SpatialDropout1D, add, concatenate
from keras.layers import CuDNNLSTM, CuDNNGRU, GlobalAveragePooling1D, GlobalMaxPooling1D, Bidirectional
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.callbacks import LearningRateScheduler, ModelCheckpoint<define_variables>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt']<define_variables>
vocab = build_vocab(df['question_text'] )
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4*LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<define_variables>
df['question_text'] = df['question_text'].apply(lambda x: x.lower() )
IDENTITY_COLUMNS = [
'transgender', 'female', 'homosexual_gay_or_lesbian', 'muslim', 'hindu',
'white', 'black', 'psychiatric_or_mental_illness', 'jewish']<define_variables>
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
AUX_COLUMNS = ['target', 'severe_toxicity','obscene','identity_attack','insult','threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<define_variables>
for index, row in df.iterrows() :
for w in paraules_prohibides:
if w in row['question_text']:
row['conte_paraula_prohibida']=1
break;
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because",
"could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not",
"don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would",
"he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will",
"how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have",
"I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will",
"i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us",
"ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not",
"oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have",
"she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have",
"so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have",
"that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is",
"here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will",
"they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have",
"wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will",
"what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have",
"when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is",
"who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have",
"you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }<define_variables>
if "sexo" in paraules_prohibides:
print("OK")
 if "casa" in paraules_prohibides:
print("OK2" )
swear_words = [
' 4r5e ',
' 5h1t ',
' 5hit ',
' a55 ',
' anal ',
' anus ',
' ar5e ',
' arrse ',
' arse ',
' ass ',
' ass-fucker ',
' asses ',
' assfucker ',
' assfukka ',
' asshole ',
' assholes ',
' asswhole ',
' a_s_s ',
' b!tch ',
' b00bs ',
' b17ch ',
' b1tch ',
' ballbag ',
' balls ',
' ballsack ',
' bastard ',
' beastial ',
' beastiality ',
' bellend ',
' bestial ',
' bestiality ',
' biatch ',
' bitch ',
' bitcher ',
' bitchers ',
' bitches ',
' bitchin ',
' bitching ',
' bloody ',
' blow job ',
' blowjob ',
' blowjobs ',
' boiolas ',
' bollock ',
' bollok ',
' boner ',
' boob ',
' boobs ',
' booobs ',
' boooobs ',
' booooobs ',
' booooooobs ',
' breasts ',
' buceta ',
' bugger ',
' bum ',
' bunny fucker ',
' butt ',
' butthole ',
' buttmuch ',
' buttplug ',
' c0ck ',
' c0cksucker ',
' carpet muncher ',
' cawk ',
' chink ',
' cipa ',
' cl1t ',
' clit ',
' clitoris ',
' clits ',
' cnut ',
' cock ',
' cock-sucker ',
' cockface ',
' cockhead ',
' cockmunch ',
' cockmuncher ',
' cocks ',
' cocksuck ',
' cocksucked ',
' cocksucker ',
' cocksucking ',
' cocksucks ',
' cocksuka ',
' cocksukka ',
' cok ',
' cokmuncher ',
' coksucka ',
' coon ',
' cox ',
' crap ',
' cum ',
' cummer ',
' cumming ',
' cums ',
' cumshot ',
' cunilingus ',
' cunillingus ',
' cunnilingus ',
' cunt ',
' cuntlick ',
' cuntlicker ',
' cuntlicking ',
' cunts ',
' cyalis ',
' cyberfuc ',
' cyberfuck ',
' cyberfucked ',
' cyberfucker ',
' cyberfuckers ',
' cyberfucking ',
' d1ck ',
' damn ',
' dick ',
' dickhead ',
' dildo ',
' dildos ',
' dink ',
' dinks ',
' dirsa ',
' dlck ',
' dog-fucker ',
' doggin ',
' dogging ',
' donkeyribber ',
' doosh ',
' duche ',
' dyke ',
' ejaculate ',
' ejaculated ',
' ejaculates ',
' ejaculating ',
' ejaculatings ',
' ejaculation ',
' ejakulate ',
' f u c k ',
' f u c k e r ',
' f4nny ',
' fag ',
' fagging ',
' faggitt ',
' faggot ',
' faggs ',
' fagot ',
' fagots ',
' fags ',
' fanny ',
' fannyflaps ',
' fannyfucker ',
' fanyy ',
' fatass ',
' fcuk ',
' fcuker ',
' fcuking ',
' feck ',
' fecker ',
' felching ',
' fellate ',
' fellatio ',
' fingerfuck ',
' fingerfucked ',
' fingerfucker ',
' fingerfuckers ',
' fingerfucking ',
' fingerfucks ',
' fistfuck ',
' fistfucked ',
' fistfucker ',
' fistfuckers ',
' fistfucking ',
' fistfuckings ',
' fistfucks ',
' flange ',
' fook ',
' fooker ',
' fuck ',
' fucka ',
' fucked ',
' fucker ',
' fuckers ',
' fuckhead ',
' fuckheads ',
' fuckin ',
' fucking ',
' fuckings ',
' fuckingshitmotherfucker ',
' fuckme ',
' fucks ',
' fuckwhit ',
' fuckwit ',
' fudge packer ',
' fudgepacker ',
' fuk ',
' fuker ',
' fukker ',
' fukkin ',
' fuks ',
' fukwhit ',
' fukwit ',
' fux ',
' fux0r ',
' f_u_c_k ',
' gangbang ',
' gangbanged ',
' gangbangs ',
' gaylord ',
' gaysex ',
' goatse ',
' God ',
' god-dam ',
' god-damned ',
' goddamn ',
' goddamned ',
' hardcoresex ',
' hell ',
' heshe ',
' hoar ',
' hoare ',
' hoer ',
' homo ',
' hore ',
' horniest ',
' horny ',
' hotsex ',
' jack-off ',
' jackoff ',
' jap ',
' jerk-off ',
' jism ',
' jiz ',
' jizm ',
' jizz ',
' kawk ',
' knob ',
' knobead ',
' knobed ',
' knobend ',
' knobhead ',
' knobjocky ',
' knobjokey ',
' kock ',
' kondum ',
' kondums ',
' kum ',
' kummer ',
' kumming ',
' kums ',
' kunilingus ',
' l3itch ',
' labia ',
' lmfao ',
' lust ',
' lusting ',
' m0f0 ',
' m0fo ',
' m45terbate ',
' ma5terb8 ',
' ma5terbate ',
' masochist ',
' master-bate ',
' masterb8 ',
' masterbat3 ',
' masterbate ',
' masterbation ',
' masterbations ',
' masturbate ',
' mo-fo ',
' mof0 ',
' mofo ',
' mothafuck ',
' mothafucka ',
' mothafuckas ',
' mothafuckaz ',
' mothafucked ',
' mothafucker ',
' mothafuckers ',
' mothafuckin ',
' mothafucking ',
' mothafuckings ',
' mothafucks ',
' mother fucker ',
' motherfuck ',
' motherfucked ',
' motherfucker ',
' motherfuckers ',
' motherfuckin ',
' motherfucking ',
' motherfuckings ',
' motherfuckka ',
' motherfucks ',
' muff ',
' mutha ',
' muthafecker ',
' muthafuckker ',
' muther ',
' mutherfucker ',
' n1gga ',
' n1gger ',
' nazi ',
' nigg3r ',
' nigg4h ',
' nigga ',
' niggah ',
' niggas ',
' niggaz ',
' nigger ',
' niggers ',
' nob ',
' nob jokey ',
' nobhead ',
' nobjocky ',
' nobjokey ',
' numbnuts ',
' nutsack ',
' orgasim ',
' orgasims ',
' orgasm ',
' orgasms ',
' p0rn ',
' pawn ',
' pecker ',
' penis ',
' penisfucker ',
' phonesex ',
' phuck ',
' phuk ',
' phuked ',
' phuking ',
' phukked ',
' phukking ',
' phuks ',
' phuq ',
' pigfucker ',
' pimpis ',
' piss ',
' pissed ',
' pisser ',
' pissers ',
' pisses ',
' pissflaps ',
' pissin ',
' pissing ',
' pissoff ',
' poop ',
' porn ',
' porno ',
' pornography ',
' pornos ',
' prick ',
' pricks ',
' pron ',
' pube ',
' pusse ',
' pussi ',
' pussies ',
' pussy ',
' pussys ',
' rectum ',
' retard ',
' rimjaw ',
' rimming ',
' s hit ',
' s.o.b.',
' sadist ',
' schlong ',
' screwing ',
' scroat ',
' scrote ',
' scrotum ',
' semen ',
' sex ',
' sh!t ',
' sh1t ',
' shag ',
' shagger ',
' shaggin ',
' shagging ',
' shemale ',
' shit ',
' shitdick ',
' shite ',
' shited ',
' shitey ',
' shitfuck ',
' shitfull ',
' shithead ',
' shiting ',
' shitings ',
' shits ',
' shitted ',
' shitter ',
' shitters ',
' shitting ',
' shittings ',
' shitty ',
' skank ',
' slut ',
' sluts ',
' smegma ',
' smut ',
' snatch ',
' son-of-a-bitch ',
' spac ',
' spunk ',
' s_h_i_t ',
' t1tt1e5 ',
' t1tties ',
' teets ',
' teez ',
' testical ',
' testicle ',
' tit ',
' titfuck ',
' tits ',
' titt ',
' tittie5 ',
' tittiefucker ',
' titties ',
' tittyfuck ',
' tittywank ',
' titwank ',
' tosser ',
' turd ',
' tw4t ',
' twat ',
' twathead ',
' twatty ',
' twunt ',
' twunter ',
' v14gra ',
' v1gra ',
' vagina ',
' viagra ',
' vulva ',
' w00se ',
' wang ',
' wank ',
' wanker ',
' wanky ',
' whoar ',
' whore ',
' willies ',
' willy ',
' xrated ',
' xxx '
]<string_transform>
for index, row in df.iterrows() :
if row['conte_paraula_prohibida']==1 & index < 20:
print(row['question_text'] )
replace_with_fuck = []
for swear in swear_words:
replace_with_fuck.append(swear)
 replace_with_fuck = '|'.join(replace_with_fuck )<string_transform>
my_embedding_matrix = {k: v for k, v in my_embedding_matrix.items() if k in vocab}
def handle_swears(text):
text = re.sub(replace_with_fuck, ' fuck ', text)
 return text
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def get_coefs(word, *arr):

 return word, np.asarray(arr,dtype='float32' )<load_pretrained>
del embed_paragram, embed_glove, vocab; gc.collect() ; time.sleep(5 )
def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )<categorify>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def build_matrix(word_index, path):

 embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<choose_model_class>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def build_model(embedding_matrix, num_aux_targets):

 words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights = [embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x)
 ])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result =Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs =words, outputs =[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<load_from_csv>
df['question_text'] = df['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 train_df['comment_text'] = train_df['comment_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['comment_text'] = test_df['comment_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
gc.collect()
gc.collect()<prepare_x_and_y>
punct = "/-'?!.,
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )<feature_engineering>
df['size'] = df['question_text'].str.len()
print(mean(df['size']))
print(median(df['size']))
print(stdev(df['size']))
print(amax(df['size']))
print(amin(df['size']))
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >=0.5, True, False )<string_transform>
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "eur","$": "usd", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
tokenizer = text.Tokenizer(filters = CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train,maxlen=MAX_LEN)
 x_test= sequence.pad_sequences(x_test, maxlen=MAX_LEN)
<define_variables>
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN]*(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])*train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights/=sample_weights.mean()<define_variables>
df['question_text'] = df['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index,f)for f in EMBEDDING_FILES], axis =-1)
 del train_df, tokenizer
gc.collect()
checkpoint_predictions = []
weights = []<train_model>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks = [
LearningRateScheduler(lambda _: 1e-3*(0.55**global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 del model
gc.collect()<save_to_csv>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id' )<set_options>
df['question_text'] = df['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
stopwords = set(STOPWORDS)
 def show_wordcloud(data, title = None):
wordcloud = WordCloud(
 background_color='white',
stopwords=stopwords,
max_words=50,
max_font_size=40,
scale=5,
random_state=1
).generate(str(data))
fig = plt.figure(1, figsize=(10,10))
plt.axis('off')
 if title:
fig.suptitle(title, fontsize=20)
 fig.subplots_adjust(top=2.3)
 plt.imshow(wordcloud)
 plt.show()<import_modules>
df['size'] = df['question_text'].str.len()
print(mean(df['size']))
print(median(df['size']))
print(stdev(df['size']))
print(amax(df['size']))
print(amin(df['size']))
df = df.drop(['size'],axis=1 )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<import_modules>
total_set_qid = df['qid'].values
total_set_X = df["question_text"].fillna("_na_" ).values
total_set_y = df['target'].values
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
from keras.losses import binary_crossentropy
from keras import backend as K<define_variables>
tokenizer = Tokenizer(num_words=None, filters='')
 tokenizer.fit_on_texts(list(total_set_X))
total_set_X = tokenizer.texts_to_sequences(total_set_X )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<categorify>
total_set_X = pad_sequences(total_set_X, maxlen=question_length)
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<compute_test_metric>
df = pd.concat([pd.DataFrame(total_set_qid),pd.DataFrame(total_set_X), pd.DataFrame(total_set_y)], axis=1, keys=["qid", "question_text", "target"] )
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1]<choose_model_class>
def embedding_matrix_creator(embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = len(word_index)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words+1, embed_size))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_model(embedding_matrix, num_aux_targets, loss_weight):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
Attention(MAX_LEN )(x),
GlobalMaxPooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model<categorify>
emb_matrix = embedding_matrix_creator(my_embedding_matrix )
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<load_from_csv>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'] )<data_type_conversions>
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()<prepare_x_and_y>
train_df, val_df = train_test_split(df[df.target[0]!=-1], test_size=0.1)
 train_X = np.array(train_df["question_text"])
 train_y = np.array(train_df["target"])
 val_X = np.array(val_df["question_text"])
 val_y = np.array(val_df["target"] )
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values
x_test = preprocess(test['comment_text'] )<string_transform>
test_df = df[df.target[0]==-1]
test_X=np.array(test_df["question_text"])
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<concatenate>
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<load_pretrained>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
with open('temporary.pickle', mode='wb')as f:
pickle.dump(x_test, f)
 del identity_columns, weights, tokenizer, train, test, x_test
gc.collect()
checkpoint_predictions = []
weights = []<choose_model_class>
def make_old_model(embedding_matrix, embed_size=300, loss='binary_crossentropy'):
inp = Input(shape=(question_length,))
x = Embedding(embedding_matrix.shape[0], embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pl = GlobalAveragePooling1D()(x)
 max_pl = GlobalMaxPooling1D()(x)
 concat = concatenate([avg_pl, max_pl])
 dense = Dense(64, activation="relu" )(concat)
 drop = Dropout(0.1 )(concat)
 output = Dense(1, activation="sigmoid" )(concat)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss=loss, optimizer=Adam(lr=0.0001), metrics=['accuracy', f1])
 return model
build_model(embedding_matrix, y_aux_train.shape[-1], loss_weight ).summary()<train_model>
def model_lstm_gru_atten(embedding_matrix, embed_size=300, loss='binary_crossentropy'):
inp = Input(shape=(question_length,))
x = Embedding(embedding_matrix.shape[0], embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1,seed=seed_nb )(x)
 x = Bidirectional(CuDNNLSTM(64, kernel_initializer=glorot_uniform(seed=seed_nb), return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(40,kernel_initializer=glorot_uniform(seed=seed_nb), return_sequences=True))(x)
 atten_1 = Attention(question_length )(x)
 atten_2 = Attention(question_length )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(16,kernel_initializer=he_uniform(seed=seed_nb), activation="relu" )(conc)
 conc = Dropout(0.1,seed=seed_nb )(conc)
 outp = Dense(1,kernel_initializer=he_uniform(seed=seed_nb), activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss=loss, optimizer=Adam(lr=0.0001), metrics=['accuracy', f1])
 return model
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1], loss_weight)
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 1e-3 *(0.6 ** global_epoch))
] 
)
 with open('temporary.pickle', mode='rb')as f:
x_test = pickle.load(f)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 del x_test
gc.collect()
weights.append(2 ** global_epoch)
 del model
gc.collect()<prepare_output>
checkpoints = ModelCheckpoint('weights.hdf5', monitor="val_f1", mode="max", verbose=True, save_best_only=True)
 reduce_lr = ReduceLROnPlateau(monitor='val_f1', factor=0.1, patience=2, verbose=1, min_lr=0.000001 )
predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )<save_to_csv>
def tweak_threshold(pred, truth):
thresholds = []
scores = []
print("Threshold: Valor")
 for thresh in np.arange(0.01, 1.01, 0.01):
thresh = np.round(thresh, 2)
 thresholds.append(thresh)
 score = f1_score(truth,(pred>thresh ).astype(int))
print(thresh, ": ", score)
 scores.append(score)
 return np.max(scores), thresholds[np.argmax(scores)]
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )<load_from_csv>
df_X = df[df.target[0]!=-1]["question_text"]
df_y = df[df.target[0]!=-1]["target"]
train_meta = np.zeros(df_y.shape)
 test_meta = np.zeros(test_X.shape[0])
 my_splits = list(StratifiedKFold(n_splits=K_FOLDS,
shuffle=True,
random_state=DATA_SPLIT_SEED ).split(df_X, df_y))
for idx,(train_idx, valid_idx)in enumerate(my_splits):
print("======== K-FOLD: {0} ==========".format(idx))
train_X = df_X.iloc[train_idx]
train_y = df_y.iloc[train_idx]
val_X = df_X.iloc[valid_idx]
val_y = df_y.iloc[valid_idx]
model = make_old_model(emb_matrix)
 history = model.fit(train_X, train_y, batch_size=batch_size, epochs=K_FOLD_EPOCHS, validation_data=[val_X, val_y], callbacks=[checkpoints, reduce_lr])
 model.load_weights('weights.hdf5')
 eval_pred = model.predict(val_X, batch_size=batch_size, verbose=1)
 test_pred = model.predict(test_X, batch_size=batch_size, verbose=1)
 train_meta[valid_idx] = eval_pred
test_meta += test_pred.reshape(-1)/ len(my_splits )
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id' )<set_options>
score_val, best_thresh = tweak_threshold(train_meta, df_y)
 print("=====================================")
 print(f"Scored {round(score_val, 4)} for threshold {best_thresh} with untreated texts on validation data" )
<import_modules><EOS> 
y_te =(np.array(test_meta)> best_thresh ).astype(np.int)
 qid = test_df["qid"].values
submit_df = pd.concat([pd.DataFrame(qid, columns=['qid']),pd.DataFrame(y_te, columns=['prediction'])], axis = 1)
 submit_df.to_csv("submission_cesc.csv", index=False )