Kaggle competition: Quora Insincere Questions Classification
x_train, x_test, y_train, y_aux_train, test = load_data(size=1)
 x_train, x_test, tokenizer = tokenize_text(x_train, x_test, pad_len = 205)
 embedding_matrix = make_embedder(tokenizer )<train_model>
import nltk, re, string
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
lstm_units = 128
num_epochs = 5
pad_length = 250
x_train, x_test, y_train, y_aux_train, test = load_data(size=1)
 x_train, x_test, tokenizer = tokenize_text(x_train, x_test, pad_len = pad_length)
 embedding_matrix = make_embedder(tokenizer)
 train_dataset, test_dataset, y_train_torch = make_train_test(x_train, x_test, y_train, y_aux_train)
 all_test_preds = []
start = time.time()
NUM_MODELS = 2
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(0 + model_idx)
 model = NeuralNet(
 embedding_matrix, y_aux_train.shape[-1],
lstm_units, len(tokenizer.word_index)+ 1)
 model.cuda()
test_preds = train_model(
 model, train_dataset, test_dataset,
output_dim=y_train_torch.shape[-1], n_epochs = num_epochs,
loss_fn=nn.BCEWithLogitsLoss(reduction='mean'))
all_test_preds.append(test_preds)
 print('Total time taken: ', round(( time.time() - start)/60,2), 'minutes')
 print()<save_to_csv>
def clean_text(text):
print(text)
 text = text.translate(string.punctuation)
 text = text.lower()
text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 print(text)
 print("")
 return text
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<define_variables>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
EMBEDDING_PATHS = ['.. /input/pickled-word-embedding/crawl-300d-2M.pkl',
'.. /input/pickled-word-embedding/glove.840B.300d.pkl']
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4*LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
IDENTITY_COLUMNS = [
'transgender', 'female', 'homosexual_gay_or_lesbian', 'muslim', 'hindu',
'white', 'black', 'psychiatric_or_mental_illness', 'jewish'
]
AUX_COLUMNS = ['target', 'severe_toxicity','obscene','identity_attack','insult','threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'<define_variables>
X_train = train_df["question_text"].fillna("na" ).values
X_test = test_df["question_text"].fillna("na" ).values
y = train_df["target"]
SYMBOLS_TO_ISOLATE = '.,?!-;*"…:—() %
SYMBOLS_TO_REMOVE = '
🍕\r🐵\xa0\ue014\t\uf818\uf04a\xad😢🐶️\uf0e0😜😎👊\u200b\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\u202a\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\x81エンジ故障\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\ufeff\u2028😉😤⛺🙂\u3000تحكسة👮💙فزط😏🍾🎉😞\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\u200a🌠🐟💫💰💎эпрд\x95🖐🙅⛲🍰🤐👆🙌\u2002💛🙁👀🙊🙉\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\x13🚬🤓\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\uf0b7\uf04c\x9f\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\u202d💤🍇\ue613小土豆🏡❔⁉\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\x9c\x9d🗑\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\u2000үսᴦᎥһͺ\u2007հ\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\uf203\uf09a\uf222\ue608\uf202\uf099\uf469\ue607\uf410\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\uf10aლڡ🐦\U0001f92f\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'
ISOLATE_DICT = {ord(c):f' {c} ' for c in SYMBOLS_TO_ISOLATE}
REMOVE_DICT = {ord(c):f'' for c in SYMBOLS_TO_REMOVE}
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
CONTRACTION_MAPPING = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def handle_punctuation(text):
text = text.translate(REMOVE_DICT)
 text = text.translate(ISOLATE_DICT)
 return text
def clean_contractions(text, mapping=CONTRACTION_MAPPING):

 specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def preprocess(x):
x = handle_punctuation(x)
 return x
def get_coefs(word, *arr):

 return word, np.asarray(arr,dtype='float32')
 def load_embeddings(path):
if path.split('.')[-1] in ['txt','vec']:
with open(path,'rb')as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 if path.split('.')[-1] =='pkl':
with open(path,'rb')as f:
return pickle.load(f)
 def build_matrix(word_index, path):

 embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<set_options>
from keras.models import Model
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D
from keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout
from keras.layers import *
from keras.models import *
from keras.preprocessing import text, sequence
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import gc
from sklearn import metrics
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def build_model(embedding_matrix, num_aux_targets):

 words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights = [embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x)
 ])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result =Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs =words, outputs =[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<load_from_csv>
maxlen = 70
max_features = 50000
embed_size = 300
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen )
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 train_df['comment_text'] = train_df['comment_text'].progress_apply(lambda x:preprocess(x))
test_df['comment_text'] = test_df['comment_text'].progress_apply(lambda x:preprocess(x))
gc.collect()<prepare_x_and_y>
def attention_3d_block(inputs):
TIME_STEPS = inputs.shape[1].value
SINGLE_ATTENTION_VECTOR = False
input_dim = int(inputs.shape[2])
 a = Permute(( 2, 1))(inputs)
 a = Reshape(( input_dim, TIME_STEPS))(a)
 a = Dense(TIME_STEPS, activation='softmax' )(a)
 if SINGLE_ATTENTION_VECTOR:
a = Lambda(lambda x: K.mean(x, axis=1))(a)
 a = RepeatVector(input_dim )(a)
 a_probs = Permute(( 2, 1))(a)
 output_attention_mul = Multiply()([inputs, a_probs])
 return output_attention_mul
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str)
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >=0.5, True, False )<string_transform>
class AttLayer(Layer):
def __init__(self, attention_dim):
self.init = initializers.get('normal')
 self.supports_masking = True
self.attention_dim = attention_dim
super(AttLayer, self ).__init__()
def build(self, input_shape):
assert len(input_shape)== 3
self.W = K.variable(self.init(( input_shape[-1], self.attention_dim)))
 self.b = K.variable(self.init(( self.attention_dim,)))
 self.u = K.variable(self.init(( self.attention_dim, 1)))
 self.trainable_weights = [self.W, self.b, self.u]
super(AttLayer, self ).build(input_shape)
 def compute_mask(self, inputs, mask=None):
return mask
def call(self, x, mask=None):
uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))
ait = K.dot(uit, self.u)
 ait = K.squeeze(ait, -1)
 ait = K.exp(ait)
 if mask is not None:
ait *= K.cast(mask, K.floatx())
 ait /= K.cast(K.sum(ait, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 ait = K.expand_dims(ait)
 weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
 return output
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[-1] )
tokenizer = text.Tokenizer(filters = CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train,maxlen=MAX_LEN)
 x_test= sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector
del embeddings_index; gc.collect()
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN]*(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])*train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights/=sample_weights.mean()<train_model>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector
del embeddings_index; gc.collect()
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index,f)for f in EMBEDDING_PATHS], axis =-1)
 print("Embedding matrix shape:", embedding_matrix.shape)
 del train_df, tokenizer
gc.collect()<train_model>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector
del embeddings_index; gc.collect()
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks = [
LearningRateScheduler(lambda _: 1e-3*(0.55**global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 del model
gc.collect()<save_to_csv>
embedding_matrix = np.concatenate(( embedding_matrix_1, embedding_matrix_2, embedding_matrix_3), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<set_options>
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42 )
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<set_options>
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
def model2() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = AttLayer(64 )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
CRAWL_EMBEDDING_PATH = '.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'
GLOVE_EMBEDDING_PATH = '.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl' 
<load_pretrained>
MODEL2 = model2()
MODEL2.summary()
batch_size = 2048
epochs = 3
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model2.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL2.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL2.save('./model2.h5' )
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path,'rb')as f:
emb_arr = pickle.load(f)
 return emb_arr<categorify>
pred_val_y_2 = MODEL2.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_2 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_2 = thresholds[0][0]
print("Best threshold: ", best_thresh_2)
 y_pred_2 = MODEL2.predict(x_test, batch_size=1024, verbose=True )
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<normalization>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=5,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [1,2,4,8,6]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<load_from_csv>
def model4() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = AttLayer(64 )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<define_variables>
MODEL4 = model4()
MODEL4.summary()
batch_size = 1536
epochs = 3
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model4.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL4.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL4.save('./model4.h5' )
symbols_to_isolate = '.,?!-;*"…:—() %
symbols_to_delete = '
🍕\r🐵😑\xa0\ue014\t\uf818\uf04a\xad😢🐶️\uf0e0😜😎👊\u200b\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\u202a\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\x81エンジ故障\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\ufeff\u2028😉😤⛺🙂\u3000تحكسة👮💙فزط😏🍾🎉😞\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\u200a🌠🐟💫💰💎эпрд\x95🖐🙅⛲🍰🤐👆🙌\u2002💛🙁👀🙊🙉\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\x13🚬🤓\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\uf0b7\uf04c\x9f\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\u202d💤🍇\ue613小土豆🏡❔⁉\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\x9c\x9d🗑\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\u2000үսᴦᎥһͺ\u2007հ\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\uf203\uf09a\uf222\ue608\uf202\uf099\uf469\ue607\uf410\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\uf10aლڡ🐦\U0001f92f\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'<define_variables>
pred_val_y_4 = MODEL4.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_4 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_4 = thresholds[0][0]
print("Best threshold: ", best_thresh_4)
 y_pred_4 = MODEL4.predict(x_test, batch_size=1024, verbose=True )
punct = "/-'?!.,
small_caps_mapping = {
"ᴀ": "a", "ʙ": "b", "ᴄ": "c", "ᴅ": "d", "ᴇ": "e", "ғ": "f", "ɢ": "g", "ʜ": "h", "ɪ": "i",
"ᴊ": "j", "ᴋ": "k", "ʟ": "l", "ᴍ": "m", "ɴ": "n", "ᴏ": "o", "ᴘ": "p", "ǫ": "q", "ʀ": "r",
"s": "s", "ᴛ": "t", "ᴜ": "u", "ᴠ": "v", "ᴡ": "w", "x": "x", "ʏ": "y", "ᴢ": "z"}
contraction_mapping = {
"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've":
"i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam",
"mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have",
"mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have",
"o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have",
"she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not",
"shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's":"this is","that'd": "that would",
"that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is",
"here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have",
"we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will",
"what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have",
"who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have",
"trump's": "trump is", "obama's": "obama is", "canada's": "canada is", "today's": "today is"}
specail_signs = { "…": "...", "₂": "2"}
specials = ["’", "‘", "´", "`"]<string_transform>
def model5() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x0 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x1 = attention_3d_block(x0)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)
 x3 = Add()([x0, x2])
 x4 = Bidirectional(CuDNNGRU(64, return_sequences=True))(x3)
 x5 = AttLayer(64 )(x4)
 x = Dropout(0.3 )(x5)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
tokenizer = TreebankWordTokenizer()
isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}
remove_dict = {ord(c):f'' for c in symbols_to_delete}
def handle_punctuation(x):
x = x.translate(remove_dict)
 x = x.translate(isolate_dict)
 return x
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def preprocess(x):
x = handle_punctuation(x)
 x = handle_contractions(x)
 x = fix_quote(x)
 return x<prepare_x_and_y>
MODEL5 = model5()
MODEL5.summary()
batch_size = 1536
epochs = 3
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model5.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL5.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL5.save('./model5.h5' )
x_train = train['comment_text'].progress_apply(lambda x:preprocess(x))
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = test['comment_text'].progress_apply(lambda x:preprocess(x))
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = 410047<string_transform>
pred_val_y_5 = MODEL5.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_5 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_5 = thresholds[0][0]
print("Best threshold: ", best_thresh_5)
 y_pred_5 = MODEL5.predict(x_test, batch_size=1024, verbose=True )
tokenizer = text.Tokenizer(num_words = max_features, filters='',lower=False )<train_model>
pred_val_y =(3*pred_val_y_2 + 4*pred_val_y_4 + 3*pred_val_y_5)/10
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )<string_transform>
y_pred =(3*y_pred_2 + 4*y_pred_4 + 3*y_pred_5)/10
y_te =(y_pred[:,0] > best_thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<categorify>
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index =False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(submit_df )