Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<choose_model_class>
import pandas as pd
import numpy as np
from sklearn.model_selection import StratifiedKFold
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import *
from keras.models import *
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.initializers import *
from keras.optimizers import *
import keras.backend as K
from keras.callbacks import *
import tensorflow as tf
import os
import gc
import re
from unidecode import unidecode
input_shape =(512, 512, 3)
 def create_model() :
model = tf.keras.Sequential([
efn.EfficientNetB5(
 input_shape=input_shape,
weights='imagenet',
include_top=False 
),
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 return model<define_variables>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )
callbacks = [get_lr_callback(BATCH_SIZE)]
step_per_epoch = count_data_items(train_filenames)/BATCH_SIZE//REPLICAS<create_dataframe>
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))
test_dataset = get_test_dataset(ordered=True )<categorify>
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna(" " ).values
X_test = test["question_text"].fillna(" " ).values
tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]
def get_dataset(filename, shuffle=False, repeat=False, return_img_name=False, augment=False, labeled=False, batch_size=32, dim=512):
dataset = tf.data.TFRecordDataset(filename, num_parallel_reads=AUTO)
 if repeat:
dataset = dataset.repeat()
if shuffle:
dataset = dataset.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
dataset = dataset.with_options(opt)
 if labeled:
dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
dataset = dataset.map(lambda example: read_unlabeled_tfrecord(example, return_img_name), num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=augment), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(batch_size * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
del train, test
gc.collect()
FOLDS = 3
IMG_SIZE = 512
BATCH_SIZE = 32
EPOCHS = 12
weights = 1/6<choose_model_class>
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
kfold = KFold(n_splits=FOLDS, shuffle=True, random_state=0 )<load_pretrained>
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_2), axis=0)
 del embedding_matrix_1, embedding_matrix_2
gc.collect()
np.shape(embedding_matrix )
TTA = 11
preds = np.zeros(( count_data_items(test_filenames),1))
for fold,(train_file, valid_file)in enumerate(kfold.split(data_filenames)) :
print(fold)
 training_dataset = get_dataset([data_filenames[x] for x in train_file], augment=True, shuffle=True, repeat=True, labeled=True)
 valid_dataset = get_dataset([data_filenames[x] for x in valid_file], augment=True, shuffle=False, repeat=False, labeled=False)
 K.clear_session()
with strategy.scope() :
model = create_model()
if fold == 3:
print("ok")
 optimizer = tf.keras.optimizers.Adam(lr=0.001)
 model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
 callbacks = [get_lr_callback(BATCH_SIZE)]
steps_per_epoch = count_data_items([data_filenames[x] for x in train_file])/BATCH_SIZE//REPLICAS
sv = tf.keras.callbacks.ModelCheckpoint('fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='min', save_freq='epoch')
 callbacks.append(sv)
 warmup_history = model.fit(training_dataset, steps_per_epoch=steps_per_epoch, validation_data=valid_dataset, epochs=12, verbose=1, callbacks=callbacks ).history
if fold == 3:
print("whatup")
 test_dataset = get_dataset(test_filenames, augment=True, repeat=True, batch_size=BATCH_SIZE*4)
 ct_valid = count_data_items(test_filenames)
 if fold == 3:
print("hello")
 STEPS = TTA * ct_valid/32/4/REPLICAS
pred = model.predict(test_dataset,steps=STEPS,verbose=1)[:TTA*ct_valid,]
preds[:, 0] += np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1 )<concatenate>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
preds = np.concatenate(preds )<prepare_output>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
preds = preds/6<create_dataframe>
def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.22 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,))(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Attention(10 )(x)
 x = Dense(100, activation="relu" )(x)
 x = Dropout(0.22 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam())
 return model
ds = get_dataset(test_filenames, augment=False, repeat=False,
labeled=False, return_img_name=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
submission = pd.DataFrame(dict(image_name=list(image_names), target=preds))
submission = submission.sort_values('image_name')
 submission = submission.set_index("image_name")
 submission.to_csv('submission.csv')
 submission.head()<predict_on_test>
kfold = StratifiedKFold(n_splits=5, random_state=10, shuffle=True)
 bestscore = []
y_test = np.zeros(( X_test.shape[0],))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = capsule()
if i == 0:print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks)
 model.load_weights(filepath)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 y_test += np.squeeze(model.predict([X_test], batch_size=1024, verbose=2)) /5
f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
bestscore.append(threshold )
<count_values>
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>np.mean(bestscore)).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )