Kaggle competition: Quora Insincere Questions Classification
result["id"] = test.id<save_to_csv>
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('spliting data')
 df_train,df_test = train_test_split(df, random_state=1)
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df_train['question_text'])
 x_test = tokenizer.texts_to_sequences(df_test['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 x_test = pad_sequences(x_test, maxlen=max_len)
 y_train = df_train['target'].values
y_test = df_test['target'].values
print(x_train.shape)
 print(y_train.shape)
result[["id","target"]].to_csv("submission.csv",index=False )<load_from_csv>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
plt.rcParams.update({'font.size': 14})
 train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub_sample = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 print(train.shape, test.shape, sub_sample.shape )<count_duplicates>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
train.duplicated().sum()<remove_duplicates>
matrixes = [embedding_matrix_glov,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_para
gc.collect()
train = train.drop_duplicates().reset_index(drop=True )<count_missing_values>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_f1s = []
def _threshold_search(self,y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def on_epoch_end(self, epoch, logs={}):
y_val = self.validation_data[1]
y_pred = self.model.predict(self.validation_data[0])
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 search_result = self._threshold_search(y_val, y_pred)
 _val_f1 = search_result['f1']
self.val_f1s.append(_val_f1)
 print('— val_f1_on_epoch_end: %f '%(_val_f1))
return
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
batch_size = int(x_train.shape[0]*K.eval(adam.lr))
print('Batch size = ',batch_size)
 def get_model(opt, trainable=False):
inp1 = Input(shape=(max_len,))
x = Embedding(num_words, dim, weights=[matrix],trainable = trainable, )(inp1)
 x = SpatialDropout1D(0.1 )(x)
 x1 = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x1)
 x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)
 x = Add()([x1,x])
 atten_1 = Attention(max_len )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([atten_1, avg_pool, max_pool])
 x = Dense(100, activation="relu" )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
 print(model.summary())
 return model
patience = 2
modelcheck = ModelCheckpoint('model.h5',save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
model = get_model(adam)
 history = model.fit(x_train,y_train,
batch_size=batch_size,
validation_split=0.2,
epochs=4,
callbacks=[modelcheck,stop,reduce, metrics])
 y_pred = model.predict(x_train,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_train, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TRAINING SET:
',classification_report(y_train,y_pred))
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))
print('Pretraining finished...Unfreezing...')
 adam = optimizers.Adam()
model = get_model(adam,True)
 model.load_weights('model.h5')
 modelcheck = ModelCheckpoint('model.h5',save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 history = model.fit(x_train,y_train,
batch_size=batch_size,
validation_split=0.2,
epochs=4,
callbacks=[modelcheck,stop,reduce, metrics])
 print('training finished...' )
train.isnull().sum()<count_missing_values>
model.load_weights('model.h5')
 y_pred = model.predict(x_train,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_train, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TRAINING SET:
',classification_report(y_train,y_pred))
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))
test.isnull().sum()<count_missing_values>
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_final=tokenizer.texts_to_sequences(df_final['question_text'])
 x_final = pad_sequences(x_final,maxlen=max_len)
 y_pred = model.predict(x_final,batch_size=batch_size,verbose=1)
 y_pred = y_pred > search_result['threshold']
y_pred = y_pred.astype(int)
 print(y_pred[:5])
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = y_pred
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )