Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<filter>
max_features = 90000
maxlen = 50
display(all_data[all_data['Fare'].isnull() ])
 display(all_data[all_data['Fare']==0] )<feature_engineering>
os.environ['PYTHONHASHSEED'] = '11'
np.random.seed(22)
 random.seed(33)
 tf.set_random_seed(44)
_ = all_data.set_value(1043,'Fare',value=0 )<define_variables>
t0 = time.time()
splits = 5<concatenate>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
def discretize_fare(val):
fare_group = pd.qcut(all_data['Fare'],splits ).sort_values().unique()
for i in range(splits):
if val in fare_group[i]:
return i+1
elif np.isnan(val):
return val<feature_engineering>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
all_data['Fare'] = all_data['Fare'].apply(discretize_fare )<data_type_conversions>
train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
all_data['Fare'] = all_data['Fare'].fillna(5 ).astype(int )<concatenate>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 trunc = 'pre'
train_X = pad_sequences(train_X, maxlen=maxlen, truncating=trunc)
 val_X = pad_sequences(val_X, maxlen=maxlen, truncating=trunc)
 test_X = pad_sequences(test_X, maxlen=maxlen, truncating=trunc)
 train_y = train_df['target'].values
val_y = val_df['target'].values
def discretize_age(val):
age_group = pd.cut(all_data['Age'],splits ).sort_values().unique()
for i in range(splits):
if val in age_group[i]:
return i+1
elif np.isnan(val):
return 0<data_type_conversions>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector
del embeddings_index; gc.collect()
all_data['Age'] = all_data['Age'].apply(discretize_age ).astype(int )<feature_engineering>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector
del embeddings_index; gc.collect()
all_data['Title'] = all_data['Name'].apply(get_title )<categorify>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector
del embeddings_index; gc.collect()
all_data['Title'] = all_data['Title'].replace(['Ms','Mlle'],'Miss')
 all_data['Title'] = all_data['Title'].replace('Mme','Mrs')
 all_data['Title'] = all_data['Title'].replace(['Don','Dona','Lady','Sir',
'Countess','Jonkheer'],'Royal')
 all_data['Title'] = all_data['Title'].replace(['Rev','Major','Col','Capt','Dr'],'Other' )<feature_engineering>
EMBEDDING_FILE = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_4 =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in word_index.items() :
if i >= max_features: continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix_4[i] = embedding_vector
del embeddings_index; gc.collect()
def impute_age(row):
pclass = row['Pclass']
title = row['Title']
age = row['Age']
if age == 0:
return int(round(all_data.loc[(all_data['Age']!=0)& 
(all_data['Pclass']==pclass)& 
(all_data['Title']==title)]['Age'].mean() ,1))
else:
return age<feature_engineering>
embedding_matrix = np.concatenate(( embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4
gc.collect()
np.shape(embedding_matrix )
all_data['Age'] = all_data.apply(impute_age,axis=1 )<rename_columns>
print(f'Done preprocessing {time.time() - t0:.1f}s' )
_ = all_data.rename({'Cabin':'Deck'},axis=1,inplace=True )<data_type_conversions>
class ExponentialMovingAverage(Callback):
def __init__(self, model, decay=0.999, mode='epoch', n=100):

 self.decay = decay
self.mode = mode
self.ema_model = clone_model(model)
 self.ema_model.set_weights(model.get_weights())
 self.n = n
if self.mode is 'batch':
self.cnt = 0
self.ema_weights = [K.get_value(w)for w in model.trainable_weights]
self.n_weights = len(self.ema_weights)
 super(ExponentialMovingAverage, self ).__init__()
def on_batch_end(self, batch, logs={}):
if self.mode is 'batch':
self.cnt += 1
if self.cnt % self.n == 0:
self.update_weights()
def on_epoch_end(self, epoch, logs={}):
if self.mode is 'epoch':
self.update_weights()
for var, w in zip(self.ema_model.trainable_weights, self.ema_weights):
K.set_value(var, w)
 def update_weights(self):
for w_old, var_new in zip(self.ema_weights, self.model.trainable_weights):
w_old +=(1 - self.decay)*(K.get_value(var_new)- w_old )
all_data['Deck'] = all_data['Deck'].fillna('N' )<categorify>
def create_rnn_model(rnn, maxlen, embedding, max_features, embed_size,
rnn_dim=64, dense1_dim=100, dense2_dim=50,
embed_trainable=False, seed=123):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding],
trainable=embed_trainable )(inp)
 x = Dense(dense1_dim, activation='relu',
kernel_initializer=glorot_uniform(seed=seed))(x)
 x = Bidirectional(rnn(rnn_dim, return_sequences=True,
kernel_initializer=glorot_uniform(seed=seed)) )(x)
 x = GlobalMaxPooling1D()(x)
 x = Dense(dense2_dim, activation='relu',
kernel_initializer=glorot_uniform(seed=seed))(x)
 x = Dense(1, activation='sigmoid',
kernel_initializer=glorot_uniform(seed=seed))(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
def cabin_to_deck(row):
return row['Deck'][0]<feature_engineering>
def f1_best(y_val, pred_val):
best_f1 = 0
best_thresh = 0
for thresh in np.linspace(0.2, 0.4, 41):
f1 = metrics.f1_score(y_val,(pred_val > thresh ).astype(int))
if f1 > best_f1:
best_f1 = f1
best_thresh = thresh
return best_f1, best_thresh
embed_ids = [list(range(300)) , list(range(300, 600)) ,
list(range(600, 900)) , list(range(900, 1200)) ]
embed_ids_dict = {1: [embed_ids[0], embed_ids[1], embed_ids[2], embed_ids[3]],
2: [embed_ids[0] + embed_ids[1],
embed_ids[0] + embed_ids[2],
embed_ids[0] + embed_ids[3],
embed_ids[1] + embed_ids[2],
embed_ids[1] + embed_ids[3],
embed_ids[2] + embed_ids[3]],
3: [embed_ids[0] + embed_ids[1] + embed_ids[2],
embed_ids[0] + embed_ids[1] + embed_ids[3],
embed_ids[0] + embed_ids[2] + embed_ids[3],
embed_ids[1] + embed_ids[2] + embed_ids[3]],
4: [embed_ids[0] + embed_ids[1] + embed_ids[2] + embed_ids[3]]}
embed_ids_lst = embed_ids_dict[2]
embed_size = 600
rnn = CuDNNGRU
embed_trainable = False
n_models = 6
epochs = 7
batch_size = 512
dense1_dim = rnn_dim = 128
dense2_dim = 2 * rnn_dim
ema_n = int(len(train_y)/ batch_size / 10)
 decay = 0.9
scores = []
pred_avg = np.zeros(( len(val_y), 1))
pred_test_avg = np.zeros(( test_df.shape[0], 1))
for i in range(n_models):
t1 = time.time()
seed = 101 + 11 * i
cols_in_use = embed_ids_lst[i % len(embed_ids_lst)]
model = create_rnn_model(rnn, maxlen, embedding_matrix[:, cols_in_use],
max_features, embed_size,
rnn_dim=rnn_dim,
dense1_dim=dense1_dim,
dense2_dim=dense2_dim,
embed_trainable=embed_trainable,
seed=seed)
 ema = ExponentialMovingAverage(model, decay=decay, mode='batch', n=ema_n)
 model.fit(train_X, train_y, batch_size=batch_size, epochs=epochs,
callbacks=[ema], verbose=0)
 m = ema.ema_model
t_per_epoch =(time.time() - t1)/ epochs
pred = m.predict([val_X])
 pred_avg += pred
pred_test = m.predict([test_X])
 pred_test_avg += pred_test
f1_one, thresh_one = f1_best(val_y, pred)
 f1_avg, thresh_avg = f1_best(val_y, pred_avg /(i + 1))
nll_one = metrics.log_loss(val_y, pred)
 nll_avg = metrics.log_loss(val_y, pred_avg /(i + 1))
auc_one = metrics.roc_auc_score(val_y, pred)
 auc_avg = metrics.roc_auc_score(val_y, pred_avg)
 print(f' n_model:{i + 1} epoch:{epochs} F1:{f1_avg:.4f} th:{thresh_avg:.3f} ' +
f'AUC:{auc_avg:.4f} NLL:{nll_avg:.4f} One:{f1_one:.4f} {auc_one:.4f} {nll_one:.4f} ' +
f'Time:{time.time() - t1:.1f}s {t_per_epoch:.1f}s/epoch')
<count_unique_values><EOS> 
pred_test_avg /= n_models
pred_test_avg =(pred_test_avg>thresh_avg ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_avg
out_df.to_csv("submission.csv", index=False)
 print(f'Done:{time.time() - t0:.1f}s' )