Kaggle competition: Kannada MNIST
train_df = train_df.append(df_obscene, ignore_index=True, sort = False )<string_transform>
%matplotlib inline
embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<statistical_test>
train_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 print(train_data.shape )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')
 print(test_data.shape )
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = LSTM(128, return_sequences = True )(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
dig_data = pd.read_csv("/kaggle/input/Kannada-MNIST/Dig-MNIST.csv")
 print(dig_data.shape )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
validate_data = train_data[55000:]
train_label = np.float32(train_data.label)
 validate_label = np.float32(validate_data.label)
 test_label = np.float32(test_data.id)
 dig_label = np.float32(dig_data.label)
 train_image = np.float32(train_data[train_data.columns[1:]])
 validate_image = np.float32(validate_data[validate_data.columns[1:]])
 test_image = np.float32(test_data[test_data.columns[1:]])
 dig_image = np.float32(dig_data[dig_data.columns[1:]] )
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))<predict_on_test>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range = 10,
horizontal_flip = False,
zoom_range = 0.15 )
pred_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<save_to_csv>
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
train_label_transform = to_categorical(train_data.iloc[:,0])
 validate_label_transform = to_categorical(validate_data.iloc[:,0])
 dig_label_transform = to_categorical(dig_data.iloc[:,0])
 print('train_label_transform shape: %s'%str(train_label_transform.shape))
print('validate_label_transform shape: %s'%str(validate_label_transform.shape))
print('dig_label_transform shape: %s'%str(dig_label_transform.shape))
%matplotlib inline
os.environ['CUDA_VISIBLE_DEVICES'] = "0"
gpu_options = K.tf.GPUOptions(per_process_gpu_memory_fraction = 1)
 config = K.tf.ConfigProto(gpu_options = gpu_options, allow_soft_placement = True)
 K.set_session(K.tf.Session(config = config))
print(os.listdir(".. /input"))
print([dev.name for dev in device_lib.list_local_devices() ] )<load_from_csv>
train_image = train_image / 255.0
validate_image = validate_image / 255.0
test_image = test_image / 255.0
dig_image = dig_image / 255.0
train_image_reshape = train_image.reshape(train_image.shape[0],28,28,1)
 validate_image_reshape = validate_image.reshape(validate_image.shape[0],28,28,1)
 test_image_reshape = test_image.reshape(test_image.shape[0],28,28,1)
 dig_image_reshape = dig_image.reshape(dig_image.shape[0],28,28,1)
 print('train_image_reshape shape %s' %str(train_image_reshape.shape))
print('validate_image_reshape shape %s' %str(validate_image_reshape.shape))
print('test_image_reshape shape %s' %str(test_image_reshape.shape))
print('dig_image_reshape shape %s' %str(dig_image_reshape.shape))
%%time
base_path = ".. /input/"
train_df = pd.read_csv(base_path+"train.csv")
 test_df = pd.read_csv(base_path+"test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
print("Train: ", train_X.shape, train_y.shape)
 print("Validation: ", val_X.shape, val_y.shape)
 print("Test :", test_X.shape )<features_selection>
model = Sequential()
model.add(Conv2D(64, kernel_size=3, activation='relu', input_shape=(28, 28, 1),padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(64, kernel_size=3, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(64, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(128, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(128, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(256, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.3))
model.add(Flatten())
 model.add(Dense(256,kernel_regularizer=regularizers.l2(0.02)))
 model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(Dropout(0.3))
model.add(Dense(256,kernel_regularizer=regularizers.l2(0.02)))
 model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(Dropout(0.3))
model.add(Dense(512,kernel_regularizer=regularizers.l2(0.02)))
 model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(Dropout(0.3))
model.add(Dense(10, activation='softmax'))
model.summary()
%%time
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 base_path = ".. /input/embeddings/"
files = {"glove": "glove.840B.300d/glove.840B.300d.txt",
"wiki_news": "wiki-news-300d-1M/wiki-news-300d-1M.vec",
"paragram": "paragram_300_sl999/paragram_300_sl999.txt"}
embedding_matrices = {}
for emb in files:
if emb=="glove":
emb_index = dict(get_coefs(*o.split(" ")) for o in open(base_path+files[emb]))
elif emb=="wiki_news":
emb_index = dict(get_coefs(*o.split(" ")) for o in open(base_path+files[emb])\
if len(o)>100)
 elif emb=="paragram":
emb_index = dict(get_coefs(*o.split(" ")) for o in open(base_path+files[emb],
encoding="utf8",
errors='ignore')\
if len(o)>100)
 all_embs = np.stack(emb_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = emb_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
embedding_matrices[emb] = embedding_matrix<choose_model_class>
BATCH_SIZE = 512
EPOCHS = 60
%%time
inp = Input(shape=(maxlen,))
inp_glove = Embedding(max_features, embed_size, weights=[embedding_matrices["glove"]] )(inp)
 inp_glove = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp_glove)
 inp_glove = GlobalMaxPool1D()(inp_glove)
 inp_wiki = Embedding(max_features, embed_size, weights=[embedding_matrices["wiki_news"]] )(inp)
 inp_wiki = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp_wiki)
 inp_wiki = GlobalMaxPool1D()(inp_wiki)
 inp_paragram = Embedding(max_features, embed_size, weights=[embedding_matrices["paragram"]] )(inp)
 inp_paragram = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp_paragram)
 inp_paragram = GlobalMaxPool1D()(inp_paragram)
 merged = concatenate([inp_glove, inp_wiki, inp_paragram])
 x = Dense(32, activation="relu" )(merged)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 f1_scores = []
threshs = np.arange(0.1, 0.9, 0.01)
 for thresh in threshs:
thresh = np.round(thresh, 2)
 f1_score = metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))
f1_scores.append(f1_score)
 plt.plot(threshs, f1_scores)
 max_fscore = np.round(f1_scores[np.argmax(f1_scores)], 3)
 max_thresh = np.round(threshs[np.argmax(f1_scores)], 3)
 plt.title("F scores at different values of thresholds | Max: {} | Thresh {}".format(max_fscore, max_thresh))<predict_on_test>
model.compile(loss='categorical_crossentropy',optimizer=Adadelta() ,metrics=['accuracy'])
 datagen.fit(train_image_reshape )
thresh = max_thresh
pred_val_y_1 = model.predict([val_X], batch_size=1024, verbose=1)
 pred_test_y_1 = model.predict([test_X], batch_size=1024, verbose=1)
<choose_model_class>
history = model.fit_generator(datagen.flow(train_image_reshape,train_label_transform, batch_size=BATCH_SIZE),
epochs = EPOCHS,
shuffle=True,
validation_data =(validate_image_reshape,validate_label_transform),
verbose = 1,
steps_per_epoch=train_image_reshape.shape[0] // BATCH_SIZE )
%%time
filter_sizes = [1, 2, 3, 5]
num_filt = 36
inp = Input(shape=(maxlen,))
embeddings = [Embedding(max_features, embed_size, weights=[embedding_matrices[emb]] )(inp)\
for emb in embedding_matrices]
embed_maxpools = []
for embed in embeddings:
embed = SpatialDropout1D(0.1 )(embed)
 embed = Reshape(( maxlen, embed_size, 1))(embed)
 maxpools = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filt, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer="he_normal", activation="elu" )(embed)
 maxpools.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
merged = Concatenate(axis=1 )(maxpools)
 merged = Flatten()(merged)
 merged = Dropout(0.2 )(merged)
 embed_maxpools.append(merged)
 embed_maxpools = Concatenate(axis=1 )(embed_maxpools)
 x = Dense(32, activation="relu" )(embed_maxpools)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 model.fit(train_X, train_y, batch_size=256, epochs=2, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 f1_scores = []
threshs = np.arange(0.1, 0.9, 0.01)
 for thresh in threshs:
thresh = np.round(thresh, 2)
 f1_score = metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))
f1_scores.append(f1_score)
 plt.plot(threshs, f1_scores)
 max_fscore = np.round(f1_scores[np.argmax(f1_scores)], 3)
 max_thresh = np.round(threshs[np.argmax(f1_scores)], 3)
 plt.title("F scores at different values of thresholds | Max: {} | Thresh {}".format(max_fscore, max_thresh))<predict_on_test>
pred_dig = history.model.predict_classes(dig_image_reshape)
 print(metrics.accuracy_score(pred_dig, np.argmax(dig_label_transform, axis = 1)))
 print(metrics.accuracy_score(pred_dig, dig_label))
thresh = max_thresh
pred_val_y_2 = model.predict([val_X], batch_size=1024, verbose=1)
 pred_test_y_2 = model.predict([test_X], batch_size=1024, verbose=1)
<set_options>
validate_labels = []
for i in validate_label_transform:
for j, val in enumerate(i):
if val == 0.:
pass
else:
validate_labels.append(j )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
pred_validate = history.model.predict_classes(validate_image_reshape)
 metrics.accuracy_score(pred_validate, np.array(validate_labels))
%%time
inp = Input(shape=(maxlen,))
inp_glove = Embedding(max_features, embed_size, weights=[embedding_matrices["glove"]] )(inp)
 inp_glove = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp_glove)
 inp_glove = Attention(maxlen )(inp_glove)
 inp_wiki = Embedding(max_features, embed_size, weights=[embedding_matrices["wiki_news"]] )(inp)
 inp_wiki = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp_wiki)
 inp_wiki = Attention(maxlen )(inp_wiki)
 inp_paragram = Embedding(max_features, embed_size, weights=[embedding_matrices["paragram"]] )(inp)
 inp_paragram = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp_paragram)
 inp_paragram = Attention(maxlen )(inp_paragram)
 merged = concatenate([inp_glove, inp_wiki, inp_paragram])
 x = Dense(32, activation="relu" )(merged)
 x = Dropout(0.1 )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 model.fit(train_X, train_y, batch_size=256, epochs=2, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 f1_scores = []
threshs = np.arange(0.1, 0.9, 0.01)
 for thresh in threshs:
thresh = np.round(thresh, 2)
 f1_score = metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))
f1_scores.append(f1_score)
 plt.plot(threshs, f1_scores)
 max_fscore = np.round(f1_scores[np.argmax(f1_scores)], 3)
 max_thresh = np.round(threshs[np.argmax(f1_scores)], 3)
 plt.title("F scores at different values of thresholds | Max: {} | Thresh {}".format(max_fscore, max_thresh))<predict_on_test>
pred_test = history.model.predict_classes(test_image_reshape )
thresh = max_thresh
pred_val_y_3 = model.predict([val_X], batch_size=1024, verbose=1)
 pred_test_y_3 = model.predict([test_X], batch_size=1024, verbose=1)
<save_to_csv>
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )
pred_test_y = 0.5*pred_test_y_1 + 0.3*pred_test_y_2 + 0.2*pred_test_y_3
pred_test_y =(pred_test_y>=max_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<define_variables>
submission['label'] = pred_test
<count_values><EOS> 
submission.to_csv("submission.csv",index=False )