Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<load_from_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<load_pretrained>
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from keras.applications.resnet50 import ResNet50
with open('.. /input/bert-prediction/predictions.pkl','rb')as f:
predictions = pickle.load(f )<prepare_output>
train_df = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 test_df = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 sample_sub = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 dig_mnist = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv' )
train['infold_pred'] = predictions['infold_pred']
train['outfold_pred'] = predictions['outfold_pred']<feature_engineering>
train_data = np.array(train_df.iloc[:, 1:])
 val_data = np.array(dig_mnist.iloc[:, 1:])
 test_data = np.array(test_df.iloc[:, 1:])
 train_labels = to_categorical(train_df.iloc[:, 0])
 val_labels = to_categorical(dig_mnist.iloc[:, 0] )
if convert_abbrev_flag:
train["text"] = train["text"].apply(lambda x: convert_abbrev_in_text(x))
test["text"] = test["text"].apply(lambda x: convert_abbrev_in_text(x))
if remove_emoji_flag:
train["text"] = train["text"].apply(lambda x: remove_emoji(x))
test["text"] = test["text"].apply(lambda x: remove_emoji(x))
if remove_URL_flag:
train['text'] = train['text'].apply(remove_URL)
 test['text'] = test['text'].apply(remove_URL)
 if remove_html_flag:
train['text'] = train['text'].apply(remove_html)
 test['text'] = test['text'].apply(remove_html)
 if remove_punct_flag:
train['text'] = train['text'].apply(remove_punct)
 test['text'] = test['text'].apply(remove_punct )<load_from_url>
rows, cols = 28, 28
train_data = train_data.reshape(train_data.shape[0], rows, cols, 1)
 val_data = val_data.reshape(val_data.shape[0], rows, cols, 1)
 test_data = test_data.reshape(test_data.shape[0], rows, cols, 1 )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
train_data = train_data.astype('float32')
 val_data = val_data.astype('float32')
 test_data = test_data.astype('float32' )
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input,Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization<categorify>
train_data /= 255.0
val_data /= 255.0
test_data /= 255.0
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if model_type=='default':
for layer in layers:
clf_output = Dense(layer, activation=activation )(clf_output)
 out = Dense(1, activation='sigmoid' )(clf_output)
 elif model_type=='dropout':
for layer in layers:
x = Dropout(Dropout_num )(clf_output)
 clf_output = Dense(layer, activation=activation )(x)
 x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 elif model_type=='GlobalAveragePooling1D':
for layer in layers:
if Dropout_num>0:
clf_output = Dropout(Dropout_num )(clf_output)
 clf_output = Dense(layer, activation=activation )(clf_output)
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model 
<choose_model_class>
batch_size = 64
epochs = 5
input_shape =(rows, cols, 1 )
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
def baseline_model() :
model = Sequential()
model.add(Conv2D(64,(3,3), strides=1, padding="same", activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(64,(3,3), strides=1, padding="same", activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=2, strides=2, padding="same"))
model.add(Dropout(0.15))
model.add(Conv2D(128,(3,3), strides=1, padding="same", activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128,(3,3), strides=1, padding="same", activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=2, strides=2, padding="same"))
model.add(Dropout(0.15))
model.add(Conv2D(256,(3,3), strides=1, padding="same", activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(256,(3,3), strides=1, padding="same", activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=2, strides=2, padding="same"))
model.add(Dropout(0.15))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dense(10, activation='softmax'))
return model
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] )
train_input = bert_encode(train.text.values, tokenizer, max_len=max_len)
 test_input = bert_encode(test.text.values, tokenizer, max_len=max_len)
 train_labels = train.target.values
print(train_input[0].shape,test_input[0].shape )<train_model>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=8,
zoom_range = 0.15,
width_shift_range=0.15,
height_shift_range=0.15,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(train_data )
%%time
checkpoint = ModelCheckpoint('{}.h5'.format(model_name), monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=epochs,
callbacks=[checkpoint],
batch_size=batch_size,
)<predict_on_test>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
model.load_weights('{}.h5'.format(model_name))
test_pred = model.predict(test_input )<save_to_csv>
history = model.fit_generator(datagen.flow(train_data, train_labels, batch_size=batch_size),
epochs = epochs, validation_data =(val_data, val_labels),
steps_per_epoch=train_data.shape[0] // batch_size,
callbacks=[learning_rate_reduction] )
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('{}.csv'.format(model_name), index=False )<drop_column>
preds = model.predict_classes(test_data )
os.remove('{}.h5'.format(model_name))<import_modules>
sample_sub.label = preds
<load_from_csv><EOS> 
sample_sub.to_csv('SubmissionFile.csv', index=False )