Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<train_model>
import numpy as np
import pandas as pd
import os
import time
from tqdm import tqdm
import matplotlib.pyplot as plt
from matplotlib import style
import seaborn as sns
import tensorflow as tf
from sklearn import metrics
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from nltk.tokenize import word_tokenize
from scikitplot.metrics import plot_confusion_matrix
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import Add, Bidirectional, CuDNNLSTM, Dense, Input, Embedding,merge, BatchNormalization, Reshape
from keras.models import Model
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 test_dataset = data.TensorDataset(x_test_torch)
 all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx + 1)
 seed_everything(SEED + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 model.cuda()
test_preds = train_model(model, train_dataset, test_dataset, output_dim=y_train_torch.shape[-1],
loss_fn=nn.BCEWithLogitsLoss(reduction='mean'))
all_test_preds.append(test_preds)
 print()<prepare_output>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train datasets shape:", train.shape)
 print("Test datasets shape:", test.shape )
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.head()<save_to_csv>
train_data, validation_data = train_test_split(train, test_size=.1, random_state=1234)
 max_length = 100
max_features = 50000
token = Tokenizer(num_words=max_features)
 token.fit_on_texts(list(np.asarray(train_data.question_text)))
 xtrain = token.texts_to_sequences(np.asarray(train_data.question_text))
xvalidate = token.texts_to_sequences(np.asarray(validation_data.question_text))
xtest = token.texts_to_sequences(np.asarray(test.question_text))
xtrain = pad_sequences(xtrain, maxlen=max_length)
 xvalidate = pad_sequences(xvalidate, maxlen=max_length)
 xtest = pad_sequences(xtest, maxlen=max_length)
 ytrain = train_data.target
yvaliate = validation_data.target
submission.to_csv('submission.csv', index=False)
 print(os.listdir("."))<import_modules>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
import time
import numpy as np
import pandas as pd
from keras.models import Model, Sequential
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
import gc
import random
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from tensorflow.keras.preprocessing import text
import shap
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
import pickle<define_variables>
em_file = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*d.split(' ')) for d in open(em_file))
all_embs = np.stack(embedding_index.values())
 em_mean, em_std = all_embs.mean() , all_embs.std()
em_size = all_embs.shape[1]
word_index = token.word_index
nb_words = min(max_features, len(word_index))
em_matrix = np.random.normal(em_mean, em_std,(nb_words, em_size))
for word, i in word_index.items() :
if i >= max_features: continue
em_v = embedding_index.get(word)
 if em_v is not None:
em_matrix[i] = em_v
absStart = time.time() ;<define_variables>

 atten_layers = 1
num_att = 10
lstm_units = 128
CRAWL_EMBEDDING_PATH = '.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'
GLOVE_EMBEDDING_PATH = '.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl'
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
MAX_FEATURES = 400000
NUM_IDENTITY_WORDS = 15 
<define_variables>
def attention_model(em_matrix, atten_layers=atten_layers, num_att=num_att, lstm_units=lstm_units):
inp = Input(shape=(max_length,))
x = Embedding(max_features, em_matrix.shape[1], weights=[em_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(lstm_units, return_sequences=True))(x)
 def attention_block(layer, atten_len=max_length, num_multi=num_att, dense_units=lstm_units*2):
atten_list = []
for _ in range(num_multi):
atten_list.append(Attention(atten_len )(layer))
add_layer = Add()(atten_list)
 add_layer = BatchNormalization()(add_layer)
 dense_layer = Dense(dense_units, activation='relu' )(add_layer)
 return Add()([add_layer, dense_layer])
 for j in range(atten_layers):
if j ==0:
x = attention_block(x)
 else:
x = Reshape(( lstm_units*2, 1))(x)
 x = attention_block(x, atten_len=lstm_units*2)
 x = Dense(64, activation='relu' )(x)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inp, out)
 model.summary()
model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer='adam')
 return model
symbols_to_isolate = '.,?!-;*"…:—() %
symbols_to_delete = '
🍕\r🐵😑\xa0\ue014\t\uf818\uf04a\xad😢🐶️\uf0e0😜😎👊\u200b\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\u202a\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\x81エンジ故障\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\ufeff\u2028😉😤⛺🙂\u3000تحكسة👮💙فزط😏🍾🎉😞\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\u200a🌠🐟💫💰💎эпрд\x95🖐🙅⛲🍰🤐👆🙌\u2002💛🙁👀🙊🙉\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\x13🚬🤓\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\uf0b7\uf04c\x9f\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\u202d💤🍇\ue613小土豆🏡❔⁉\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\x9c\x9d🗑\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\u2000үսᴦᎥһͺ\u2007հ\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\uf203\uf09a\uf222\ue608\uf202\uf099\uf469\ue607\uf410\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\uf10aლڡ🐦\U0001f92f\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'<string_transform>
def att_new_model(em_matrix, atten_layers=atten_layers, num_att=num_att, lstm_units=lstm_units):
inp = Input(shape=(max_length,))
x = Embedding(max_features, em_matrix.shape[1], weights=[em_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(lstm_units, return_sequences=True))(x)
 def attention_block(layer, atten_len=max_length, num_multi=num_att, dense_units=lstm_units*2):
add_layer_list = []
for j in range(num_att):
atten_list = list()
for _ in range(num_multi):
atten_list.append(Attention(atten_len )(layer))
add_layer_list.append(Add()(atten_list))
add_layer = concatenate(add_layer_list)
 add_layer = BatchNormalization()(add_layer)
 dense_layer = Dense(dense_units, activation='relu' )(add_layer)
 return dense_layer
x = attention_block(x)
 x = Dense(64, activation='relu' )(x)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inp, out)
 model.summary()
model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer='adam')
 return model
tokenizer = TreebankWordTokenizer()
isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}
remove_dict = {ord(c):f'' for c in symbols_to_delete}
def handle_punctuation(x):
x = x.translate(remove_dict)
 x = x.translate(isolate_dict)
 return x
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def preprocess(x):
x = handle_punctuation(x)
 x = handle_contractions(x)
 x = fix_quote(x)
 return x<concatenate>
model = attention_model(em_matrix=em_matrix, atten_layers=1, lstm_units=64, num_att=10)
 model.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)
 pred_vali_glove = model.predict(xvalidate, batch_size=1024)
 best_thre = evaluate(yvaliate, pred_vali_glove)
 pred_test_glove = model.predict(xtest )
def get_balanced_set(df, FOCUS_COLUMN, size = 0):
if(size == 0):
size = df.loc[df[FOCUS_COLUMN] == True].shape[0] * 2
true_df = df.loc[df[FOCUS_COLUMN] == True][:round(size/2)]
false_df = df.loc[df[FOCUS_COLUMN] == False][:round(size/2)]
joined = pd.concat([true_df, false_df])
 return joined<load_pretrained>
del embedding_index, all_embs, word_index, em_matrix
gc.collect()
time.sleep(10 )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path,'rb')as f:
emb_arr = pickle.load(f)
 return emb_arr
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( MAX_FEATURES + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= MAX_FEATURES:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model 
<load_from_csv>
em_file = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(em_file, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embedding_index.values())
 em_mean, em_std = all_embs.mean() , all_embs.std()
em_size = all_embs.shape[1]
word_index = token.word_index
nb_words = min(max_features, len(word_index))
em_matrix = np.random.normal(em_mean, em_std,(nb_words, em_size))
for word, i in word_index.items() :
if i >= max_features: continue
em_v = embedding_index.get(word)
 if em_v is not None:
em_matrix[i] = em_v
chunkStart = time.time()
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 print('CHUNK', time.time() - chunkStart)
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 print('CHUNK', time.time() - chunkStart)
 train_df[TEXT_COLUMN] = train_df[TEXT_COLUMN].apply(lambda x:preprocess(x))
print('CHUNK', time.time() - chunkStart)
 test_df[TEXT_COLUMN] = test_df[TEXT_COLUMN].apply(lambda x:preprocess(x))
gc.collect()
print('CHUNK', time.time() - chunkStart)
 print('OVERALL', time.time() - absStart )<feature_engineering>
model = attention_model(em_matrix=em_matrix)
 model.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)
 pred_vali_para = model.predict(xvalidate, batch_size=1024)
 best_thre = evaluate(yvaliate, pred_vali_para)
 pred_test_para = model.predict(xtest )
class TextPreprocessor(object):
def __init__(self, vocab_size):
self._vocab_size = vocab_size
self._tokenizer = None
self._embedding_matrix = None
def create_tokenizer(self, text_list):
tokenizer = text.Tokenizer(num_words = MAX_FEATURES, filters='',lower=False)
 crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
self._tokenizer = tokenizer
self._embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 del crawl_matrix
del glove_matrix
gc.collect()
def transform_text(self, text_list):
print(text_list[:10])
 text_matrix = self._tokenizer.texts_to_matrix(text_list)
 return text_matrix 
<choose_model_class>
del embedding_index, all_embs, word_index, em_matrix
gc.collect()
time.sleep(10 )
def get_top_words(vals, num_words, word_index):
means = np.matrix(vals[0] ).mean(0)
 means = np.absolute(means)
 words = set()
while len(words)< num_words:
idx = means.argmax()
idx
words.add(word_index[idx])
 means[0, idx] = -1000
return words
def create_identity_model(vocab_size, num_tags):
model = Sequential()
model.add(Dense(50, input_shape=(vocab_size,), activation='relu'))
model.add(Dense(25, activation='relu'))
model.add(Dense(num_tags, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<prepare_x_and_y>
lr = LinearRegression()
model_num = 2
pred_data = np.empty([len(pred_vali_para), model_num])
 pred_data[:, 0] = pred_vali_glove.reshape(-1,)
 pred_data[:, 1] = pred_vali_para.reshape(-1,)
 lr.fit(pred_data, yvaliate)
 weights = lr.coef_
sub_pred_weighted = np.sum([pred_data[:, i]*weights[i] for i in range(model_num)], axis=0)
 best_thre = evaluate(yvaliate ,sub_pred_weighted )
def get_words_for_identity(identity_df, IDENTITY_COLUMN, VOCAB_SIZE = 400):
id_train = get_balanced_set(identity_df, IDENTITY_COLUMN)
 tokenizer = text.Tokenizer(num_words = VOCAB_SIZE, filters='',lower=False)
 tokenizer.fit_on_texts(id_train['comment_text'])
 body_train = tokenizer.texts_to_matrix(id_train['comment_text'])
 id_model = create_identity_model(VOCAB_SIZE, 1)
 id_model.fit(
 x=body_train,
y=id_train[IDENTITY_COLUMN],
epochs=10,
batch_size=128,
validation_split=0.1,
verbose=0)
 words = tokenizer.word_index
word_lookup = list()
for i in words.keys() :
word_lookup.append(i)
 word_lookup = [''] + word_lookup
num_explainers = 2000
explainer_idx = np.random.randint(body_train.shape[0], size=num_explainers)
 attrib_data = body_train[explainer_idx,:]
explainer = shap.DeepExplainer(id_model, attrib_data)
 num_explanations = 1000
explanations_idx = np.random.randint(body_train.shape[0], size=num_explanations)
 shap_vals = explainer.shap_values(body_train[explanations_idx,:])
 top_words = get_top_words(shap_vals, NUM_IDENTITY_WORDS, word_lookup)
 return top_words<define_variables>
sub_data = np.empty([len(xtest), model_num])
 sub_data[:, 0] = pred_test_glove.reshape(-1,)
 sub_data[:, 1] = pred_test_para.reshape(-1, )
<string_transform><EOS> 
sub_pred = np.sum([sub_data[:, i]*weights[i] for i in range(model_num)], axis=0)
 sub_pred =(sub_pred > best_thre ).astype(int)
 sub_df = pd.DataFrame({'qid':test.qid.values})
 sub_df['prediction'] = sub_pred
sub_df.to_csv('submission.csv', index=False)