Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<load_from_csv>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 x_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
<import_modules>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
import matplotlib.pyplot as plt
import tensorflow as tf
import cv2
from collections import deque
import numpy as np<prepare_x_and_y>
train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 embed_size = 300
max_features = 95000
maxlen = 80
train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
y_train = train['label']
x_train = train.drop(labels = ['label'], axis=1 )<train_model>
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
x_train = x_train.values.reshape(x_train.shape[0], 28, 28, 1)
 x_test = x_test.values.reshape(x_test.shape[0], 28, 28, 1 )<define_variables>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
input_shape =(28, 28, 1)
<data_type_conversions>
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
x_train = x_train.astype('float32')
 x_test = x_test.astype('float32' )<categorify>
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='glorot_uniform', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',f1] )
x_train /= 255
x_test /= 255 
<import_modules>
model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Dropout, Flatten, MaxPooling2D<choose_model_class>
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))
model = Sequential()
model.add(Conv2D(256, kernel_size=(5, 5), activation='relu', input_shape=input_shape))
model.add(Conv2D(128, kernel_size=(5, 5), activation='relu'))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(100, activation=tf.nn.relu))
model.add(Dropout(0.2))
model.add(Dense(10, activation=tf.nn.softmax))<choose_model_class>
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'] )<train_model>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
model.fit(x=x_train,y=y_train,epochs=150, verbose = 1 )<predict_on_test>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
pred = model.predict(x_test)
 results = np.argmax(pred,axis = 1)
 results = pd.Series(results,name = 'Label' )<save_to_csv>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
sub = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 sub.to_csv("cnn_best_model.csv",index=False )<define_variables>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
final_models = {"keras1":"", "keras2":""}<prepare_x_and_y>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
train_data = pd.read_csv(".. /input/train.csv")
 x_train_all = np.array(train_data.drop(["label"], axis=1))
print(x_train_all[0][x_train_all[0] > 0])
 x_train_all = x_train_all/255
x_train_all = stretch_image(x_train_all)
 y_train_all = np.array(train_data["label"])
 idx_all = range(x_train_all.shape[0])
 if 1==2:
bad_data_idx = [28290, 16301,14101,15065, 6389,7764,28611,20954,2316, 37056, 37887, 36569, 40257]
plot_bad_data(x_train_all[bad_data_idx], y_train_all[bad_data_idx])
 x_train_all = np.delete(x_train_all, bad_data_idx, axis=0)
 y_train_all = np.delete(y_train_all, bad_data_idx)
 idx_all = np.delete(idx_all, bad_data_idx)
 y_train_all = keras.utils.to_categorical(y_train_all, num_classes=10)
 x_train, x_valid, y_train, y_valid, idx_train, idx_valid = skm.train_test_split(x_train_all, y_train_all,idx_all, test_size=0.2)
 test_data = pd.read_csv(".. /input/test.csv")
 x_test = np.array(test_data)
 x_test = x_test/255
x_test = stretch_image(x_test)
<train_model>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
horizontal_flip=False, shear_range=0.2)
 datagen.fit(x_train.reshape(-1,28,28,1)) 
<choose_model_class>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
learning_rate_reduction = keras.callbacks.ReduceLROnPlateau(monitor='acc',
patience=1,
verbose=1,
factor=0.5,
min_lr=0.000001 )<train_on_grid>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
def tensorflow_keras_model(x_train, y_train, x_valid, y_valid, num_classes,\
num_epochs, learning_rate):
keras_model = keras.models.Sequential()
keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(6,6),strides=(1,1), \
padding="same",input_shape=(28,28,1)))
 keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(6,6),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Flatten())
 keras_model.add(keras.layers.Dense(units=1024))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Dense(units=num_classes))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('softmax'))
opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 keras_model.compile(optimizer=opt,loss="categorical_crossentropy", metrics=["accuracy"])
 keras_model.fit(x_train.reshape(-1,28,28,1), y_train, epochs=num_epochs, callbacks=[learning_rate_reduction])
 final_models["keras1"] = keras_model
keras_model.save("Model_MNIST_Keras_Lenet.h5")
 cur_y_pred = keras_model.predict(x_valid.reshape(-1,28,28,1))
y_valid_argmax = np.argmax(y_valid, 1)
 y_pred_argmax = np.argmax(cur_y_pred, 1)
 y_correct = np.equal(y_valid_argmax, y_pred_argmax)
 acc = y_correct.sum() /y_pred_argmax.shape[0]
return cur_y_pred, acc, y_valid_argmax,y_pred_argmax<train_on_grid>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
def tensorflow_keras_model_2(x_train, y_train, x_valid, y_valid, num_classes,\
num_epochs, learning_rate):
keras_model = keras.models.Sequential()
keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(3,3),strides=(1,1), \
padding="same",activation='relu', input_shape=(28,28,1)))
 keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(3,3),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(5,5),strides=(2,2), \
padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(3,3),strides=(1,1), \
padding="same",activation='relu',input_shape=(28,28,1)))
 keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(3,3),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(5,5),strides=(2,2), \
padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.050))
keras_model.add(keras.layers.Conv2D(filters=128,kernel_size=(3,3),strides=(1,1), \
padding="same",activation='relu'))
keras_model.add(keras.layers.Conv2D(filters=128,kernel_size=(3,3),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Conv2D(filters=128,kernel_size=(5,5),strides=(2,2), \
padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.075))
keras_model.add(keras.layers.Flatten())
 keras_model.add(keras.layers.Dense(units=1024))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.100))
keras_model.add(keras.layers.Dense(units=128))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.100))
keras_model.add(keras.layers.Dense(units=num_classes))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('softmax'))
opt = keras.optimizers.RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0)
 keras_model.compile(optimizer=opt,loss="categorical_crossentropy", metrics=["accuracy"])
 keras_model.fit_generator(datagen.flow(x_train.reshape(-1,28,28,1), y_train, batch_size=32),steps_per_epoch=len(x_train)/ 32, \
epochs=num_epochs, callbacks=[learning_rate_reduction])
 final_models["keras2"] = keras_model
keras_model.save("Model_MNIST_Keras_Resnet.h5")
 cur_y_pred = keras_model.predict(x_valid.reshape(-1,28,28,1))
y_valid_argmax = np.argmax(y_valid, 1)
 y_pred_argmax = np.argmax(cur_y_pred, 1)
 y_correct = np.equal(y_valid_argmax, y_pred_argmax)
 acc = y_correct.sum() /y_pred_argmax.shape[0]
return cur_y_pred, acc, y_valid_argmax,y_pred_argmax , keras_model 
<train_model>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
model2_list = []
num_model = 12
def exec_tensorflow_keras_model_2() :
num_epochs = 30
learning_rate=0.00001
num_rows, num_features, num_classes = x_train.shape[0], x_train.shape[1], 10
for imodel in range(num_model):
final_pred_base_model, acc,y_valid_argmax,y_pred_argmax, keras_model = \
tensorflow_keras_model_2(x_train, y_train, x_valid, y_valid, num_classes, num_epochs, learning_rate)
 model2_list.append(keras_model)
 print("Num Epoch:", num_epochs, " Accuracy:", acc)
 plot_y_true_vs_y_pred(x_valid, y_valid_argmax.reshape(len(y_valid)) , y_pred_argmax.reshape(len(y_valid)) , idx_valid )<train_on_grid>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
def tensorflow_keras_model_3(x_train, y_train, x_valid, y_valid, num_classes,\
num_epochs, learning_rate):
keras_model = keras.models.Sequential()
keras_model.add(keras.layers.Conv2D(filters=6,kernel_size=(6,6),strides=(1,1), \
padding="same",input_shape=(28,28,1)))
 keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=16,kernel_size=(6,6),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=120,kernel_size=(6,6),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Flatten())
 keras_model.add(keras.layers.Dense(units=120))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Dense(units=120))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Dense(units=num_classes))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('softmax'))
opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 keras_model.compile(optimizer=opt,loss="categorical_crossentropy", metrics=["accuracy"])
 keras_model.fit(x_train.reshape(-1,28,28,1), y_train, epochs=num_epochs, callbacks=[learning_rate_reduction])
 final_models["keras1"] = keras_model
keras_model.save("Model_MNIST_Keras_Lenet.h5")
 cur_y_pred = keras_model.predict(x_valid.reshape(-1,28,28,1))
y_valid_argmax = np.argmax(y_valid, 1)
 y_pred_argmax = np.argmax(cur_y_pred, 1)
 y_correct = np.equal(y_valid_argmax, y_pred_argmax)
 acc = y_correct.sum() /y_pred_argmax.shape[0]
return cur_y_pred, acc, y_valid_argmax,y_pred_argmax<predict_on_test>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
model2 = final_models["keras2"]
x_valid = x_valid.reshape(-1,28,28,1)
 y_pred_valid2 = np.zeros(( x_valid.shape[0],10))
for imodel in range(num_model):
y_pred_valid2 = y_pred_valid2 + model2_list[imodel].predict(x_valid)
 y_pred_valid = y_pred_valid2
y_pred_valid_final = np.argmax(y_pred_valid, axis=1)
 y_correct = np.equal(y_pred_valid_final, np.argmax(y_valid, axis=1))
acc_final = y_correct.sum() /y_valid.shape[0]
print(acc_final )<predict_on_test>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
x_test = x_test.reshape(-1,28,28,1)
 y_pred_test2 = np.zeros(( x_test.shape[0],10))
for imodel in range(num_model):
y_pred_test2 = y_pred_test2 + model2_list[imodel].predict(x_test)
<prepare_output>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
y_pred_test = y_pred_test2
y_pred_test_final = np.argmax(y_pred_test, axis=1)
<save_to_csv>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
dictionary_data = {"ImageId":np.arange(1, x_test.shape[0]+1), "Label":y_pred_test_final}
df_final = pd.DataFrame(dictionary_data)
 df_final.to_csv("submission.csv", index=False )<load_from_csv>
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
train = pd.read_csv(".. /input/train.csv")
 print(train.shape)
 train.head()<load_from_csv>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
test= pd.read_csv(".. /input/test.csv")
 print(test.shape)
 test.head()<prepare_x_and_y>
pred_val_y =(4 * pred_glove_val_y + pred_fasttext_val_y + 3 * pred_paragram_val_y + 2 * pred_cnn_val_y)/ 10.0
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1 )<train_model>
pred_test_y =(4 * pred_glove_test_y + pred_fasttext_test_y + 3 * pred_paragram_test_y + 2 * pred_cnn_test_y)/ 10.0
pred_test_y =(pred_test_y > best_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )