Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, BatchNormalization,Conv1D,Dropout,MaxPooling1D,Flatten, Input
from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
import tensorflow_addons as tfa
import tensorflow.keras.backend as k
def clean_abbreviation(df, abbreviations):
compiled_abbreviation = re.compile('(%s)' % '|'.join(abbreviations.keys()))
def replace(match):
return abbreviations[match.group(0)]
df['question_text'] = df["question_text"].apply(
 lambda x: _clean_abreviation(x, compiled_abbreviation, replace)
)
 return df
def _clean_abreviation(x, compiled_re, replace):
specials = ["’", "‘", "´", "`"]
for s in specials:
x = x.replace(s, "'")
 return compiled_re.sub(replace, x )<define_variables>
def preprocess(df):
df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df['cp_time'] = df['cp_time'].astype(np.float32)
 df.drop(['sig_id'],axis = 1, inplace = True)
 return df
spells = {
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu.be': 'youtube',
'youtu ': 'youtube ',
'qoura': 'quora',
'sallary': 'salary',
'Whta': 'what',
'whta': 'what',
'narcisist': 'narcissist',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'ethereum',
'etherium': 'ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k': '2000',
'2k10': '2010',
'2k11': '2011',
'2k12': '2012',
'2k13': '2013',
'2k14': '2014',
'2k15': '2015',
'2k16': '2016',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'exgirlfriend': 'ex girlfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'quorans': 'quora user',
'quoran': 'quora user',
'pokémon': 'pokemon',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium',
'pubg': 'video game',
'dota': 'video game',
'dota2': 'video game',
'fortnite': 'video game',
'league of legends': 'video game'
}<feature_engineering>
train = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_targets = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 train_targets.drop('sig_id',axis = 1, inplace = True )
def clean_spells(df, spells):
compiled_spells = re.compile('(%s)' % '|'.join(spells.keys()))
def replace(match):
return spells[match.group(0)]
df['question_text'] = df["question_text"].apply(
 lambda x: _clean_spells(x, compiled_spells, replace)
)
 return df
def _clean_spells(x, compiled_re, replace):
return compiled_re.sub(replace, x )<categorify>
train = preprocess(train)
 test = preprocess(test )
def clean_numbers(df):
df['question_text'] = df['question_text'].apply(lambda x: _clean_numbers(x))
return df
def _clean_numbers(x):
if bool(re.search(r'\d', x)) :
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<define_variables>
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
train["g-effect"] = abs(train[GENES] ).sum(axis=1)
 train["g-effect"]=train["g-effect"]/len(GENES)
 train["c-effect"] = abs(train[CELLS] ).sum(axis=1)
 train["c-effect"]=train["g-effect"]/len(CELLS)
 train["effect"] = train["g-effect"]+ train["c-effect"]
test["g-effect"] = abs(test[GENES] ).sum(axis=1)
 test["g-effect"] = test["g-effect"]/len(GENES)
 test["c-effect"] = abs(test[CELLS] ).sum(axis=1)
 test["c-effect"] =test["g-effect"]/len(CELLS)
 test["effect"] = test["g-effect"]+ test["c-effect"]
all_puncts={',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '*', '+', '\',
'•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′',
'█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓',
'—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━',
'⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○',
'◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐',
'├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\x97', '↺',
'∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊',
'┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´',
'↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\x93', '≧', '］',
'\x94', '∀', '♛', '\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％',
'╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭',
'┊', '╯', '☾', '➔', '∴', '\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐',
'▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\x91', '➚', '◡', '╰', '\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒',
'☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜',
'‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢',
'≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '⎯', '↠', '۩', '☰', '◥',
'⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ',
'✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦',
'◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ',
'⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋',
'､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞',
'◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ',
'✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱',
'☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊',
'➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹',
'⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚',
'☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛',
'⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙',
'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘',
'˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛',
'☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑',
'✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ',
'⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘',
'⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊',
'⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷',
'◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝',
'⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢',
'≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢'}
glove_puncs = set("/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi'}
remove_punc = all_puncts - glove_puncs
del all_puncts 
<categorify>
train['g_diff'] = train[GENES].max(axis = 1)- train[GENES].min(axis = 1)
 train['c_diff'] = train[CELLS].max(axis = 1)- train[CELLS].min(axis = 1)
 test['g_diff'] = test[GENES].max(axis = 1)- test[GENES].min(axis = 1)
 test['c_diff'] = test[CELLS].max(axis = 1)- test[CELLS].min(axis = 1 )
def clean_and_map_special_chars(df, punct, mapping, removes):
df["question_text"] = df["question_text"].apply(lambda x: _clean_and_map_special_chars(x, punct, mapping, removes))
return df
def _clean_and_map_special_chars(text, punct, mapping, removes):
for r in removes:
if r in text:
text = text.replace(r, '')
 for p in punct:
if p in text:
text = text.replace(p, f' {p} ')
 for m in mapping:
if m in text:
text = text.replace(m, mapping[m])
 specials = {'\u200b': ' ', '…': '', '\ufeff': '', 'करना': '', 'है': '',
'\x7f':'', '\xa0':'', '\ufeff':'', '\u200e':'', '\u202a':'',
'\u202c':'', '\u2060':'', '\uf0d8':'', '\ue019':'', '\uf02d':'',
'\u200f':'', '\u2061':'', '\ue01b':'', '
':' ', '\t':' ' }
for s in specials:
text = text.replace(s, specials[s])
 return text<feature_engineering>
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
n_comp = 70
data = pd.concat([pd.DataFrame(train[GENES]), pd.DataFrame(test[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[GENES]))
train2 = data2[:train.shape[0]]; test2 = data2[-test.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train = pd.concat(( train, train2), axis=1)
 test = pd.concat(( test, test2), axis=1)
 n_comp = 20
data = pd.concat([pd.DataFrame(train[CELLS]), pd.DataFrame(test[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[CELLS]))
train2 = data2[:train.shape[0]]; test2 = data2[-test.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train = pd.concat(( train, train2), axis=1)
 test = pd.concat(( test, test2), axis=1 )
def clean_space(df):
compiled_re = re.compile(r"\s+")
 df['question_text'] = df["question_text"].apply(lambda x: _clean_space(x, compiled_re))
return df
def _clean_space(x, compiled_re):
return compiled_re.sub(" ", x )<drop_column>
for c in train.columns:
try:
train[c+"_bin"] = pd.cut(train[c],bins = 100, labels = False)
 test[c+"_bin"] = pd.cut(test[c],bins = 100, labels = False)
 except:
print(c )
def clean(df):
df = clean_lower(df)
 print('Lower is done')
 df = clean_abbreviation(df, abbreviations)
 print('Abbreviation is done')
 df = clean_spells(df, spells)
 print('Spells is done')
 df = clean_and_map_special_chars(df, glove_puncs, punct_mapping, remove_punc)
 print('Special Chars is done')
 df = clean_numbers(df)
 print('Remove numbers is done')
 df = clean_space(df)
 print('Spaces is done')
 return df<load_pretrained>
top_feats = [0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 14, 15,
16, 18, 19, 20, 21, 23, 24, 25, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 44, 45, 46,
48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76,
78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 92,
93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 105, 106, 107,
108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120,
121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 132, 133, 134,
135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
149, 150, 151, 152, 153, 154, 155, 157, 159, 160, 161, 163, 164,
165, 166, 167, 168, 169, 170, 172, 173, 175, 176, 177, 178, 180,
181, 182, 183, 184, 186, 187, 188, 189, 190, 191, 192, 193, 195,
197, 198, 199, 202, 203, 205, 206, 208, 209, 210, 211, 212, 213,
214, 215, 218, 219, 220, 221, 222, 224, 225, 227, 228, 229, 230,
231, 232, 233, 234, 236, 238, 239, 240, 241, 242, 243, 244, 245,
246, 248, 249, 250, 251, 253, 254, 255, 256, 257, 258, 259, 260,
261, 263, 265, 266, 268, 270, 271, 272, 273, 275, 276, 277, 279,
282, 283, 286, 287, 288, 289, 290, 294, 295, 296, 297, 299, 300,
301, 302, 303, 304, 305, 306, 308, 309, 310, 311, 312, 313, 315,
316, 317, 320, 321, 322, 324, 325, 326, 327, 328, 329, 330, 331,
332, 333, 334, 335, 338, 339, 340, 341, 343, 344, 345, 346, 347,
349, 350, 351, 352, 353, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 368, 369, 370, 371, 372, 374, 375, 376, 377,
378, 379, 380, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391,
392, 393, 394, 395, 397, 398, 399, 400, 401, 403, 405, 406, 407,
408, 410, 411, 412, 413, 414, 415, 417, 418, 419, 420, 421, 422,
423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435,
436, 437, 438, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450,
452, 453, 454, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465,
466, 468, 469, 471, 472, 473, 474, 475, 476, 477, 478, 479, 482,
483, 485, 486, 487, 488, 489, 491, 492, 494, 495, 496, 500, 501,
502, 503, 505, 506, 507, 509, 510, 511, 512, 513, 514, 516, 517,
518, 519, 521, 523, 525, 526, 527, 528, 529, 530, 531, 532, 533,
534, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547,
549, 550, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563,
564, 565, 566, 567, 569, 570, 571, 572, 573, 574, 575, 577, 580,
581, 582, 583, 586, 587, 590, 591, 592, 593, 595, 596, 597, 598,
599, 600, 601, 602, 603, 605, 607, 608, 609, 611, 612, 613, 614,
615, 616, 617, 619, 622, 623, 625, 627, 630, 631, 632, 633, 634,
635, 637, 638, 639, 642, 643, 644, 645, 646, 647, 649, 650, 651,
652, 654, 655, 658, 659, 660, 661, 662, 663, 664, 666, 667, 668,
669, 670, 672, 674, 675, 676, 677, 678, 680, 681, 682, 684, 685,
686, 687, 688, 689, 691, 692, 694, 695, 696, 697, 699, 700, 701,
702, 703, 704, 705, 707, 708, 709, 711, 712, 713, 714, 715, 716,
717, 723, 725, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738,
739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751,
752, 753, 754, 755, 756, 758, 759, 760, 761, 762, 763, 764, 765,
766, 767, 769, 770, 771, 772, 774, 775, 780, 781, 782, 783, 784,
785, 787, 788, 790, 793, 795, 797, 799, 800, 801, 805, 808, 809,
811, 812, 813, 816, 819, 820, 821, 822, 823, 825, 826, 827, 829,
831, 832, 833, 834, 835, 837, 838, 839, 840, 841, 842, 844, 845,
846, 847, 848, 850, 851, 852, 854, 855, 856, 858, 860, 861, 862,
864, 867, 868, 870, 871, 873, 874]
datadir = '.. /input/quora-insincere-questions-classification'
train_df, test_df = load_data(datadir )<load_pretrained>
top_feats += [i for i in range(-195,0,1)]
features = train.columns[top_feats].unique()
target_cols = [x for x in submission.columns if x != 'sig_id']
glovedir = os.path.join('.. /input/quora-insincere-questions-classification', 'embeddings', 'glove.840B.300d', 'glove.840B.300d.txt')
 embed_glove = load_embed(glovedir )<concatenate>
def create_model1() :
model = Sequential()
model.add(Input(train[features].shape[1],))
model.add(BatchNormalization())
 model.add(Dropout (.4144839109739807))
model.add(tfa.layers.WeightNormalization(Dense(2048,activation='relu')))
 model.add(BatchNormalization())
 model.add(Dropout (.4144839109739807))
model.add(tfa.layers.WeightNormalization(Dense(512,activation='relu')))
 model.add(BatchNormalization())
 model.add(Dropout (.4144839109739807))
model.add(tfa.layers.WeightNormalization(Dense(4096,activation='relu')))
 model.add(BatchNormalization())
 model.add(tfa.layers.WeightNormalization(Dense(len(train_targets.columns), activation='sigmoid')))
 model.compile(loss='binary_crossentropy', optimizer = tfa.optimizers.Lookahead(tf.optimizers.Adam(1e-1)) , metrics=['BinaryCrossentropy'])
 return model
add_lower(embed_glove,
pd.concat(( train_df['question_text'], test_df['question_text']),axis=0, ignore_index=True))<drop_column>
def create_model2() :
model = Sequential()
model.add(Input(train[features].shape[1],))
model.add(BatchNormalization())
 model.add(tfa.layers.WeightNormalization(Dense(2048,activation='relu')))
 model.add(BatchNormalization())
 model.add(Dropout (.1))
model.add(tfa.layers.WeightNormalization(Dense(512,activation='relu')))
 model.add(BatchNormalization())
 model.add(Dropout (.1))
model.add(tfa.layers.WeightNormalization(Dense(4096,activation='relu')))
 model.add(BatchNormalization())
 model.add(Dropout (.1))
model.add(tfa.layers.WeightNormalization(Dense(len(train_targets.columns), activation='sigmoid')))
 model.compile(loss='binary_crossentropy', optimizer = tfa.optimizers.Lookahead(tf.optimizers.Adam(1e-1)) , metrics=['BinaryCrossentropy'])
 return model
train_df = clean(train_df)
 test_df = clean(test_df )<split>
def log_loss_numpy(y_pred,y_true):
loss = 0
y_pred_clip = np.clip(y_pred.values, 1e-15, 1 - 1e-15)
 for i in range(y_pred.shape[1]):
y = np.log(1 - y_pred_clip[:, i])
 loss += - np.mean(y_true[:, i] * np.log(y_pred_clip[:, i])+(1 - y_true[:, i])* np.log(1 - y_pred_clip[:, i]))
print(loss / y_pred.shape[1] )
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2019 )<string_transform>
def log_loss_numpy2(y_pred,y_true):
loss = 0
y_pred_clip = np.clip(y_pred.values,.001,.9999)
 for i in range(y_pred.shape[1]):
y = np.log(1 - y_pred_clip[:, i])
 loss += - np.mean(y_true[:, i] * np.log(y_pred_clip[:, i])+(1 - y_true[:, i])* np.log(1 - y_pred_clip[:, i]))
print(loss / y_pred.shape[1] )
embed_size = 300
max_features = 85000
maxlen = 64
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 word_index = tokenizer.word_index
train_y = train_df['target'].values
val_y = val_df['target'].values 
<categorify>
def get_architecture(T):
if T==0:
model = create_model1()
elif T==1:
model = create_model2()
return model
def make_embed_matrix(embeddings_index, word_index, max_features):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size+2))
for word, i in word_index.items() :
if i < max_features:
embedding_vector = embeddings_index.get(word)
 word_sent = TextBlob(word ).sentiment
extra_embed = [word_sent.polarity,word_sent.subjectivity]
if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector,extra_embed)
 return embedding_matrix<drop_column>
f_preds = []
f_res = []
for T in range(2):
test_preds = submission.copy()
test_preds[target_cols] = 0
res = train_targets.copy()
res[target_cols] = 0
num_folds = 10
n_seeds = [10,14,16,77,76,34,21]
for seed in(n_seeds):
print(f'model {T} Seed {seed}')
 for n,(tr, te)in enumerate(KFold(n_splits=num_folds, random_state=seed, shuffle=True ).split(train)) :
k.clear_session()
model = get_architecture(T)
 es = EarlyStopping(monitor = 'val_loss', min_delta = 0.0001, patience = 4, mode = 'min',
baseline = None, restore_best_weights = True, verbose = 0)
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=0, epsilon=1e-4, mode='min')
 checkpoint_path = 'model.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 model.fit(train[features].astype(float ).values[tr],
train_targets.astype(float ).values[tr],
validation_data=(train[features].values[te], train_targets.astype(float ).values[te]),
epochs=30, batch_size=128,
callbacks=[reduce_lr_loss,cb_checkpt,es], verbose=0 
)
 model.load_weights('model.hdf5')
 test_preds.loc[:,target_cols] += model.predict(test[features].values)
 res.loc[te, target_cols] += model.predict(train[features].values[te])
 del model
test_preds.loc[:,target_cols]/=(len(n_seeds)*num_folds)
 res.loc[:, train_targets.columns] /=(len(n_seeds))
log_loss_numpy(res,train_targets.values)
 f_preds.append(test_preds)
 f_res.append(res )
embedding_matrix = make_embed_matrix(embed_glove, word_index, max_features)
 del word_index
del embed_glove
del tokenizer<compute_test_metric>
def blend_func(w,preds,f_preds):
weight = [w['weight1'],w['weight2']]
s = sum(weight)
 f_sub = submission.copy()
f_sub.loc[:,target_cols] = 0
blend = train_targets.copy()
blend.loc[:,target_cols] = 0
for n,i in enumerate(preds):
blend.loc[:,target_cols]+=i.values*weight[n]/s
blend.loc[train['cp_type']==1, train_targets.columns] = 0
print(log_loss_numpy(blend,train_targets.values))
print(log_loss_numpy2(blend,train_targets.values))
for n,i in enumerate(f_preds):
f_sub.loc[:,target_cols]+=i.loc[:,target_cols]*weight[n]/s
return f_sub
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<choose_model_class>
weights = {'weight1': 1.7026867550445766, 'weight2': 3.7348807669139044}
f_sub = blend_func(weights,f_res,f_preds )
<init_hyperparams><EOS> 
f_sub.loc[:,target_cols] = np.clip(f_sub.loc[:,target_cols],.001,.9999)
 f_sub.loc[test['cp_type']==1, target_cols] = 0
f_sub.to_csv('submission.csv', index=False )