Kaggle competition: Kannada MNIST
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<choose_model_class>
import tensorflow as tf
from sklearn.model_selection import train_test_split
import keras
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from keras.layers import LeakyReLU
from keras.layers.normalization import BatchNormalization
from tensorflow.keras.callbacks import LearningRateScheduler
from tensorflow.keras.callbacks import EarlyStopping
import matplotlib.pyplot as plt
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test = pd.read_csv("/kaggle/input/Kannada-MNIST/test.csv")
 Dig_MNIST = pd.read_csv("/kaggle/input/Kannada-MNIST/Dig-MNIST.csv" )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
sample_sub = pd.read_csv("/kaggle/input/Kannada-MNIST/sample_submission.csv" )
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
X=train.iloc[:,1:].values
Y=train.iloc[:,0].values
Y[:10]
def model_gru_srk_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<choose_model_class>
X_train, X_valid, Y_train, Y_valid = train_test_split(X, Y, test_size = 0.10, random_state=42 )
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
train_datagen = ImageDataGenerator(rescale = 1./255.,
rotation_range = 10,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 0.1,
zoom_range = 0.25,
horizontal_flip = False )
def model_gru_atten_3(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<find_best_params>
valid_datagen = ImageDataGenerator(rescale=1./255 )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<split>
def lr_decay(epoch):
return initial_learningrate * 0.99 ** epoch
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index )<define_variables>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)) ,
tf.keras.layers.BatchNormalization(momentum=0.5, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation='softmax')
 ])
 model.summary()
outputs = []<predict_on_test>
initial_learningrate=2e-3
batch_size = 1024
epochs = 50
input_shape =(28, 28, 1 )
pred_val_y, pred_test_y, best_score = train_pred(model_gru_atten_3(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '3 GRU w/ atten'] )<predict_on_test>
es = EarlyStopping(monitor='val_loss', verbose=1, patience=10 )
pred_val_y, pred_test_y, best_score = train_pred(model_gru_srk_atten(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'gru atten srk'] )<predict_on_test>
model.compile(loss="categorical_crossentropy",
optimizer=RMSprop(lr=initial_learningrate),
metrics=['accuracy'] )
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN'] )<predict_on_test>
from sklearn import metrics
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN GloVe'] )<predict_on_test>
history = model.fit_generator(
 train_datagen.flow(X_train,Y_train, batch_size=batch_size),
steps_per_epoch=100,
epochs=epochs,
callbacks=[LearningRateScheduler(lr_decay),
es
],
validation_data=valid_datagen.flow(X_valid,Y_valid),
validation_steps=50,
verbose=2 )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU'] )<predict_on_test>
preds_dig=model.predict_classes(x_dig/255)
 metrics.accuracy_score(preds_dig, y_dig )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention'] )<predict_on_test>
keras.backend.clear_session()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention GloVe'] )<predict_on_test>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)) ,
tf.keras.layers.BatchNormalization(momentum=0.5, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation='softmax')
 ])
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention Para'] )<sort_values>
initial_learningrate=2e-3
batch_size = 1024
epochs = 50
input_shape =(28, 28, 1 )
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
print(weights )<compute_test_metric>
model.compile(loss="categorical_crossentropy",
optimizer=RMSprop(lr=initial_learningrate),
metrics=['accuracy'] )
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
history = model.fit_generator(
 train_datagen.flow(X,Y, batch_size=batch_size),
steps_per_epoch=100,
epochs=epochs,
callbacks=[LearningRateScheduler(lr_decay)
 ],
validation_data=valid_datagen.flow(X_valid,Y_valid),
validation_steps=50,
verbose=2 )
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<string_transform>
preds_dig=model.predict_classes(x_dig/255)
 metrics.accuracy_score(preds_dig, y_dig )
eng_stopwords = set(stopwords.words("english")) 
<load_from_csv>
predictions = model.predict_classes(x_test/255.)
train_df = pd.read_csv(".. /input/train.csv")
 gc.collect()<load_from_csv>
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )
test_df = pd.read_csv(".. /input/test.csv" )<count_missing_values>
submission['label'] = predictions
train_df["question_text"].isna().sum() , test_df["question_text"].isna().sum() ,<split>
submission.to_csv("submission.csv",index=False )