Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Add, Concatenate, Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, Convolution2D, Reshape
from keras.layers import Bidirectional, GlobalMaxPool1D, MaxPool1D, GlobalMaxPool2D, Flatten, ZeroPadding1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
games['ID'] = games.apply(lambda r: '_'.join(map(str, [r['Season']]+sorted([r['WTeamID'],r['LTeamID']]))), axis=1)
 games['IDTeams'] = games.apply(lambda r: '_'.join(map(str, sorted([r['WTeamID'],r['LTeamID']]))), axis=1)
 games['Team1'] = games.apply(lambda r: sorted([r['WTeamID'],r['LTeamID']])[0], axis=1)
 games['Team2'] = games.apply(lambda r: sorted([r['WTeamID'],r['LTeamID']])[1], axis=1)
 games['IDTeam1'] = games.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team1']])) , axis=1)
 games['IDTeam2'] = games.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team2']])) , axis=1 )<categorify>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
games['Team1Seed'] = games['IDTeam1'].map(seeds ).fillna(0)
 games['Team2Seed'] = games['IDTeam2'].map(seeds ).fillna(0 )<feature_engineering>
train_df, val_df = train_test_split(train_df, test_size=0.05, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 60
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
games['ScoreDiff'] = games['WScore'] - games['LScore']
games['Pred'] = games.apply(lambda r: 1.if sorted([r['WTeamID'],r['LTeamID']])[0]==r['WTeamID'] else 0., axis=1)
 games['ScoreDiffNorm'] = games.apply(lambda r: r['ScoreDiff'] * -1 if r['Pred'] == 0.else r['ScoreDiff'], axis=1)
 games['SeedDiff'] = games['Team1Seed'] - games['Team2Seed']
games = games.fillna(-1 )<feature_engineering>
def getGloVeEmbeddings() :
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def getFastTextEmbeddings() :
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def getParagramEmbeddings() :
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
sub['WLoc'] = 3
sub['Season'] = sub['ID'].map(lambda x: x.split('_')[0])
 sub['Season'] = sub['ID'].map(lambda x: x.split('_')[0])
 sub['Season'] = sub['Season'].astype(int)
 sub['Team1'] = sub['ID'].map(lambda x: x.split('_')[1])
 sub['Team2'] = sub['ID'].map(lambda x: x.split('_')[2])
 sub['IDTeams'] = sub.apply(lambda r: '_'.join(map(str, [r['Team1'], r['Team2']])) , axis=1)
 sub['IDTeam1'] = sub.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team1']])) , axis=1)
 sub['IDTeam2'] = sub.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team2']])) , axis=1)
 sub['Team1Seed'] = sub['IDTeam1'].map(seeds ).fillna(0)
 sub['Team2Seed'] = sub['IDTeam2'].map(seeds ).fillna(0)
 sub['SeedDiff'] = sub['Team1Seed'] - sub['Team2Seed']
sub = sub.fillna(-1 )<merge>
glove_embedding = getGloVeEmbeddings()
fasttext_embedding = getFastTextEmbeddings()
paragram_embedding = getParagramEmbeddings()
games = pd.merge(games, gb, how='left', left_on='IDTeams', right_on='IDTeams_c_score')
 sub = pd.merge(sub, gb, how='left', left_on='IDTeams', right_on='IDTeams_c_score' )<define_variables>
class KMaxPooling(Layer):

 def __init__(self, k=1, axis=1, **kwargs):
super(KMaxPooling, self ).__init__(**kwargs)
 self.input_spec = InputSpec(ndim=3)
 self.k = k
assert axis in [1,2], 'expected dimensions(samples, filters, convolved_values),\
cannot fold along samples dimension or axis not in list [1,2]'
self.axis = axis
self.transpose_perm = [0,1,2]
self.transpose_perm[self.axis] = 2
self.transpose_perm[2] = self.axis
def compute_output_shape(self, input_shape):
input_shape_list = list(input_shape)
 input_shape_list[self.axis] = self.k
return tuple(input_shape_list)
 def call(self, x):
transposed_for_topk = tf.transpose(x, perm=self.transpose_perm)
 top_k = tf.nn.top_k(transposed_for_topk, k=self.k, sorted=True, name=None)[0]
transposed_back = tf.transpose(top_k, perm=self.transpose_perm)
 return transposed_back
class Folding(Layer):
def __init__(self, **kwargs):
super(Folding, self ).__init__(**kwargs)
 self.input_spec = InputSpec(ndim=3)
 def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[1], int(input_shape[2]/2))
def call(self, x):
input_shape = x.get_shape().as_list()
splits = tf.split(x, num_or_size_splits=int(input_shape[2]/2), axis=2)
 reduce_sums = [tf.reduce_sum(split, axis=2)for split in splits]
row_reduced = tf.stack(reduce_sums, axis=2)
 return row_reduced
def BiRNN() :
inp = Input(shape=(maxlen,))
glove = Embedding(max_features, embed_size, weights=[glove_embedding] )(inp)
 fast = Embedding(max_features, embed_size, weights=[fasttext_embedding] )(inp)
 paragram = Embedding(max_features, embed_size, weights=[paragram_embedding] )(inp)
 x = Add()([glove, fast, paragram])
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 return model
def MGNC_CNN() :
inp = Input(shape=(maxlen,))
glove = Embedding(max_features, embed_size, weights=[glove_embedding] )(inp)
 fast = Embedding(max_features, embed_size, weights=[fasttext_embedding] )(inp)
 paragram = Embedding(max_features, embed_size, weights=[paragram_embedding] )(inp)
 filter_sizes = [3,5]
conv_pools = []
for text_embedding in [glove,fast,paragram]:
for filter_size in filter_sizes:
l_zero = ZeroPadding1D(( filter_size-1,filter_size-1))(text_embedding)
 l_conv = Conv1D(filters=16, kernel_size=filter_size, padding='same', activation='tanh' )(l_zero)
 l_pool = GlobalMaxPool1D()(l_conv)
 conv_pools.append(l_pool)
 l_merge = Concatenate(axis=1 )(conv_pools)
 l_dense = Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.01))(l_merge)
 l_out = Dense(1, activation='sigmoid' )(l_dense)
 model = Model(inputs=inp, outputs=l_out)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
 print(model.summary())
 return model
def MV_CNN() :
inp = Input(shape=(maxlen,))
glove = Embedding(max_features, embed_size, weights=[glove_embedding] )(inp)
 fast = Embedding(max_features, embed_size, weights=[fasttext_embedding] )(inp)
 paragram = Embedding(max_features, embed_size, weights=[paragram_embedding] )(inp)
 k_top = 4
filter_sizes = [3,5]
layer_1 = []
for text_embedding in [glove, fast, paragram]:
conv_pools = []
for filter_size in filter_sizes:
l_zero = ZeroPadding1D(( filter_size-1,filter_size-1))(text_embedding)
 l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='tanh' )(l_zero)
 l_pool = KMaxPooling(k=30, axis=1 )(l_conv)
 conv_pools.append(( filter_size,l_pool))
layer_1.append(conv_pools)
 last_layer = []
for layer in layer_1:
for(filter_size, input_feature_maps)in layer:
l_zero = ZeroPadding1D(( filter_size-1,filter_size-1))(input_feature_maps)
 l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='tanh' )(l_zero)
 l_pool = KMaxPooling(k=k_top, axis=1 )(l_conv)
 last_layer.append(l_pool)
 l_merge = Concatenate(axis=1 )(last_layer)
 l_flat = Flatten()(l_merge)
 l_dense = Dense(128, activation='relu' )(l_flat)
 l_out = Dense(1, activation='sigmoid' )(l_dense)
 model = Model(inputs=inp, outputs=l_out)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
 print(model.summary())
 return model
col = [c for c in games.columns if c not in ['ID', 'DayNum', 'ST', 'Team1', 'Team2', 'IDTeams', 'IDTeam1', 'IDTeam2', 'WTeamID', 'WScore', 'LTeamID', 'LScore', 'NumOT', 'Pred', 'ScoreDiff', 'ScoreDiffNorm', 'WLoc'] + c_score_col]<compute_train_metric>
mvcnn = MV_CNN()
mvcnn.fit(train_X, train_y, batch_size=1024, epochs=2, validation_data=(val_X, val_y))
model = ExtraTreesClassifier(n_estimators=200)
 model.fit(games[col].fillna(-1), games['Pred'])
 predictions = model.predict(games[col].fillna(-1)).clip(0,1)
 print('Log Loss:', log_loss(games['Pred'], predictions))<save_to_csv>
pred_y = mvcnn.predict([val_X], batch_size=1024, verbose=1)
 max_f1 = 0
max_thresh = 0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_y > thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, score))
if score > max_f1:
max_f1 = score
max_thresh = thresh
print("** Best F1 score at threshold {0} is {1}".format(max_thresh, max_f1))
pred_mgnccnn = pred_y
sub['Pred'] = model.predict(sub[col].fillna(-1)).clip(1,0)
 sub[['ID', 'Pred']].to_csv('submission_et.csv', index=False )<define_variables>
pred_y = mvcnn.predict([test_X], batch_size=1024, verbose=1)
 pred_test_y =(pred_y > max_thresh ).astype(int)
 out_df = pd.DataFrame({"qid": test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )