Kaggle competition: Kannada MNIST
model.load_weights(filepath)
 pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=2)
 pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=2)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_glove_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 pred_test_y =(pred_glove_test_y>best_thresh ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )<import_modules>
rand_seed = 53
%env PYTHONHASHSEED=0
np.random.seed(rand_seed)
 rn.seed(rand_seed)
 tf.config.threading.set_inter_op_parallelism_threads(1)
 tf.config.threading.set_intra_op_parallelism_threads(1)
 tf.random.set_seed(rand_seed )
tqdm.pandas()<define_variables>
def read_data(file_name):
file_path = '.. /input/Kannada-MNIST/' + file_name
data_df = pd.read_csv(file_path)
 pixels_df = data_df.drop(columns='label')
 pixels_array = pixels_df.to_numpy(dtype=np.uint8)
 reshaped_pixels_array = pixels_array.reshape(-1, 28, 28, 1)
 labels_array = data_df.label.values
return(reshaped_pixels_array, labels_array )
embedding_size = 300
max_features = 95000
max_len = 100<categorify>
X, y = read_data('train.csv')
 X_train, X_test, y_train, y_test = \
train_test_split(X, y, test_size=0.1, random_state=rand_seed, stratify=y)
 print(X_train.shape)
 print(y_train.shape)
 print(X_test.shape)
 print(y_test.shape )
def clean_text(x):
x = str(x)
 for p in string.punctuation + '“…':
x = x.replace(p, ' ' + p + ' ')
 x = x.replace('_', '')
 x = re.sub("`","'", x)
 x = re.sub("(?i)n't",' not', x)
 x = re.sub("(?i)'re",' are', x)
 x = re.sub("(?i)'s",' is', x)
 x = re.sub("(?i)'d",' would', x)
 x = re.sub("(?i)'ll",' will', x)
 x = re.sub("(?i)'t",' not', x)
 x = re.sub("(?i)'ve",' have', x)
 x = re.sub("(?i)'m",' am', x)
 x = re.sub("(?i)n\’t",' not', x)
 x = re.sub("(?i)\’re",' are', x)
 x = re.sub("(?i)\’s",' is', x)
 x = re.sub("(?i)\’d",' would', x)
 x = re.sub("(?i)\’ll",' will', x)
 x = re.sub("(?i)\’t",' not', x)
 x = re.sub("(?i)\’ve",' have', x)
 x = re.sub("(?i)\’m",' am', x)
 x = re.sub('(?i)Quorans','Quora', x)
 x = re.sub('(?i)Qoura','Quora', x)
 x = re.sub('(?i)Quoran','Quora', x)
 x = re.sub('(?i)dropshipping','drop shipping', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 x = re.sub('(?i)WooCommerce','Commerce', x)
 x = re.sub('(?i)hairfall','hair fall', x)
 x = re.sub('(?i)marksheet','mark sheet', x)
 x = re.sub('(?i)articleship','article ship', x)
 x = re.sub('(?i)cryptocurrencies','cryptocurrency', x)
 x = re.sub('(?i)coinbase','cryptocurrency', x)
 x = re.sub('(?i)altcoin','bitcoin', x)
 x = re.sub('(?i)altcoins','bitcoins', x)
 x = re.sub('(?i)litecoin','bitcoin', x)
 x = re.sub('(?i)litecoins','bitcoins', x)
 x = re.sub('(?i)demonetisation','demonetization', x)
 x = re.sub('(?i)ethereum','bitcoin', x)
 x = re.sub('(?i)ethereums','bitcoins', x)
 x = re.sub('(?i)quorans','quora', x)
 x = re.sub('(?i)Brexit','britan exit', x)
 x = re.sub('(?i)upwork','freelance', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)Blockchain','blockchain', x)
 x = re.sub('(?i)GDPR','General Data Protection Regulation', x)
 x = re.sub('(?i)Qoura','quora', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Cryptocurrency','cryptocurrency', x)
 x = re.sub('(?i)Binance','cryptocurrency', x)
 x = re.sub('(?i)Redmi','mobile phone', x)
 x = re.sub('(?i)TensorFlow','Tensor Flow', x)
 x = re.sub('(?i)Golang','programming language', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 return x<compute_test_metric>
label_count = len(unique_y)
 print(label_count )
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32' )<choose_model_class>
def make_conv_layer(filter_size, suffix, inputs):
x = Conv2D(
 filter_size, kernel_size=(3, 3), padding='same', activation='relu',
kernel_initializer=he_normal(seed=rand_seed), name='conv_' + suffix )(inputs)
 outputs = BatchNormalization(name='bn_' + suffix )(x)
 return outputs
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<define_search_model>
def make_model() :
inputs = Input(shape=(28, 28, 1), name="input")
 x = Lambda(lambda v: v / 255.0, name='scaling' )(inputs)
 x = make_conv_layer(64, '1_1', x)
 x = make_conv_layer(64, '1_2', x)
 x = make_conv_layer(64, '1_3', x)
 x = MaxPooling2D(pool_size=(2, 2), name="maxpool_1" )(x)
 x = SpatialDropout2D(0.25, name='sp_dpout_1' )(x)
 x = make_conv_layer(128, '2_1', x)
 x = make_conv_layer(128, '2_2', x)
 x = make_conv_layer(128, '2_3', x)
 x = MaxPooling2D(pool_size=(2, 2), name='maxpool_2' )(x)
 x = SpatialDropout2D(0.25, name='sp_dpout_2' )(x)
 x = make_conv_layer(256, '3_1', x)
 x = make_conv_layer(256, '3_2', x)
 x = make_conv_layer(256, '3_3', x)
 x = GlobalAveragePooling2D(name='gblavgpool' )(x)
 x = Dense(256, activation='relu', name='dense' )(x)
 x = Dropout(0.25, name='dropout' )(x)
 outputs = Dense(label_count, activation='softmax', name='outputs' )(x)
 model = Model(inputs=inputs, outputs=outputs)
 model.compile(
 optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
 return model
def get_model_1(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(80, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Conv1D(64, kernel_size=2, padding='valid', kernel_initializer='he_uniform' )(x)
 y = Dense(16, activation='relu' )(y)
 y = Flatten()(y)
 atten = Attention(max_len )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([atten, avg_pool, max_pool])
 conc = Dense(64, activation='relu' )(conc)
 conc_z = concatenate([conc, y])
 conc_z = Dense(32, activation='relu' )(conc_z)
 conc_z = Dropout(0.1 )(conc_z)
 output = Dense(1, activation='sigmoid' )(conc_z)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
train_image_generator = ImageDataGenerator(
 rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=10,
zoom_range=0.2,
fill_mode='constant',
cval=0,
data_format='channels_last')
 test_image_generator = ImageDataGenerator()
def get_model_2(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(max_len )(x)
 output = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
batch_size = 256
steps_per_epoch =(X_train.shape[0] + batch_size - 1)// batch_size
train_image_flow = train_image_generator.flow(
 X_train, y_train, batch_size=batch_size, shuffle=True, seed=rand_seed)
 test_image_flow = test_image_generator.flow(
 X_test, y_test, batch_size=batch_size, shuffle=True, seed=rand_seed)
 print("batch_size: {0}".format(batch_size))
print("steps_per_epoch: {0}".format(steps_per_epoch))
def get_model_3(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 42
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Reshape(( max_len, embedding_size, 1))(x)
 conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embedding_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 maxpool_0 = MaxPool2D(pool_size=(max_len - filter_sizes[0] + 1, 1))(conv_0)
 maxpool_1 = MaxPool2D(pool_size=(max_len - filter_sizes[1] + 1, 1))(conv_1)
 maxpool_2 = MaxPool2D(pool_size=(max_len - filter_sizes[2] + 1, 1))(conv_2)
 maxpool_3 = MaxPool2D(pool_size=(max_len - filter_sizes[3] + 1, 1))(conv_3)
 z = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 output = Dense(1, activation='sigmoid' )(z)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<choose_model_class>
def get_lr(model):
return K.get_value(model.optimizer.lr)
 def set_lr(model, lr):
K.set_value(model.optimizer.lr, lr )
def get_model_4(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)
 atten = Attention(max_len )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([atten, avg_pool, max_pool])
 conc = Dense(64, activation='relu' )(conc)
 conc = Dropout(0.1 )(conc)
 output = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
def find_lr_on_batch_end(model, logs, lr_list, loss_list, lr_mult):
lr = get_lr(model)
 lr_list.append(lr)
 loss = logs['loss']
loss_list.append(loss)
 set_lr(model, lr * lr_mult )
def get_model_5(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(max_len )(x)
 x = Dense(16, activation='relu' )(x)
 x = Dropout(0.1 )(x)
 output = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
find_lr_start_lr = 1e-10
find_lr_end_lr = 1.0
find_lr_epochs = 3
find_lr_total_batch_count = find_lr_epochs * steps_per_epoch
find_lr_lr_mult =(find_lr_end_lr / find_lr_start_lr)**(1.0 / find_lr_total_batch_count)
 print("find_lr_epochs: {0}".format(find_lr_epochs))
print("find_lr_total_batch_count: {0}".format(find_lr_total_batch_count))
print("find_lr_lr_mult: {0}".format(find_lr_lr_mult))
def get_model_6(embedding_matrix):
inp = Input(shape=(max_len,))
x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.15 )(x)
 x = Bidirectional(CuDNNLSTM(max_len, return_sequences=True))(x)
 x = Conv1D(64, kernel_size=2, padding='valid', kernel_initializer='he_uniform' )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 output = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<compute_test_metric>
find_lr_model = make_model()
find_lr_lr_list = []
find_lr_loss_list = []
find_lr_callback = LambdaCallback(
 on_batch_end=lambda batch, logs: find_lr_on_batch_end(
 find_lr_model, logs, find_lr_lr_list, find_lr_loss_list, find_lr_lr_mult))
def get_f1_metric(y_true, y_predicted):
f1_metrics = []
for thresh in np.arange(0.1, 0.51, 0.01):
thresh = np.round(thresh, 2)
 f1_metric = f1_score(y_true,(y_predicted>thresh ).astype(int))
f1_metrics.append(( thresh, f1_metric))
print('F1 score at threshold {0} is {1}'.format(thresh, f1_metric))
threshold = max(f1_metrics, key=lambda x: x[1])[0]
print(threshold)
 return threshold<load_from_csv>
set_lr(find_lr_model, find_lr_start_lr)
 find_lr_history = find_lr_model.fit_generator(
 train_image_flow,
steps_per_epoch=steps_per_epoch,
epochs=find_lr_epochs,
validation_data=test_image_flow,
callbacks=[find_lr_callback],
verbose=2 )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print('Train shape : ', train_df.shape)
 print('Test shape : ', test_df.shape )<feature_engineering>
train_epochs = 150
def lr_schedule(epoch_index, current_lr):
if epoch_index == 0:
new_lr = 1e-3
elif epoch_index == 49:
new_lr = 3e-4
elif epoch_index == 99:
new_lr = 1e-4
else:
new_lr = current_lr
if new_lr != current_lr:
print(
 "Epoch {0}: Learning late changed from {1:.5f} to {2:.5f}".format(
 epoch_index + 1, current_lr, new_lr))
return new_lr
lr_scheduler = LearningRateScheduler(lr_schedule, verbose=0 )
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))<split>
best_model_file_name = "best_model.hdf5"
model_check_point = ModelCheckpoint(
 best_model_file_name, monitor='val_sparse_categorical_accuracy', mode='max',
verbose=0, save_best_only=True, save_weights_only=True, period=1 )
train_df, val_df = train_test_split(train_df,
test_size=0.07,
random_state=42,
stratify=train_df['target'])
 X_train = train_df['question_text'].values
y_train = train_df['target'].values
X_val = val_df['question_text'].values
y_val = val_df['target'].values
X_test = test_df['question_text'].values
print(X_train.shape,
y_train.shape,
X_val.shape,
y_val.shape,
X_test.shape )<string_transform>
train_history = train_model.fit_generator(
 train_image_flow,
steps_per_epoch=steps_per_epoch,
epochs=train_epochs,
validation_data=test_image_flow,
callbacks=[lr_scheduler, model_check_point],
verbose=2 )
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_val))
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 X_train = pad_sequences(X_train, maxlen=max_len)
 X_val = pad_sequences(X_val, maxlen=max_len)
 X_test = pad_sequences(X_test, maxlen=max_len)
 word_index = tokenizer.word_index
print(X_train.shape, X_val.shape, X_test.shape, len(word_index))<feature_engineering>
best_train_model = make_model()
best_train_model.load_weights(best_model_file_name )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix_glove = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in tqdm(word_index.items()):
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_glove[i] = embedding_vector
del embeddings_index
gc.collect()
print(embedding_matrix_glove.shape )<define_variables>
train_result = best_train_model.evaluate(X_test, y_test)
 print(train_result )
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix_wiki = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in tqdm(word_index.items()):
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_wiki[i] = embedding_vector
del embeddings_index, word_index
gc.collect()
print(embedding_matrix_wiki.shape )<compute_test_metric>
test_df = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_df.head()
embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_wiki], axis = 0)
 np.shape(embedding_matrix )<train_model>
test_pixels_df = test_df.drop(columns='id')
 test_pixels_array = test_pixels_df.to_numpy(dtype=np.uint8)
 test_images = test_pixels_array.reshape(-1, 28, 28, 1)
 print(test_images.shape )
model_1 = get_model_1(embedding_matrix_glove)
 model_1.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))<train_model>
model_preds = best_train_model.predict(test_images)
 print(model_preds.shape )
model_2 = get_model_2(embedding_matrix_glove)
 model_2.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))<train_model>
sample_submission_df = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 sample_submission_df.head()
model_3 = get_model_3(embedding_matrix)
 model_3.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))<train_model>
sample_submission_df['label'] = pred_labels
sample_submission_df.head()
model_4 = get_model_4(embedding_matrix)
 model_4.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))<train_model>
sample_submission_df.to_csv('submission.csv', index=False)
 print('Done!' )