Kaggle competition: Kannada MNIST
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow import keras
from keras.layers import Flatten
from keras.layers import Conv2D
from keras.layers import MaxPool2D
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import BatchNormalization
from keras.layers import Average
from keras.layers import Activation
from tensorflow.keras import layers
def encode_text(texts, tokenizer, max_len):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
<feature_engineering>
def load_data(do_reshape=False):
DMNIST_pd = pd.read_csv("/kaggle/input/Kannada-MNIST/Dig-MNIST.csv")
 train_pd = pd.read_csv("/kaggle/input/Kannada-MNIST/train.csv")
 test_pd = pd.read_csv("/kaggle/input/Kannada-MNIST/test.csv")
 x_train_np, y_train_np = train_pd.iloc[:,1:].to_numpy() , train_pd.iloc[:,0].to_numpy()
x_DMNIST_np, y_DMNIST_np = DMNIST_pd.iloc[:,1:].to_numpy() , DMNIST_pd.iloc[:,0].to_numpy()
x_data=np.concatenate(( x_train_np,x_DMNIST_np))
y_data=np.concatenate(( y_train_np,y_DMNIST_np))
x_test_full = test_pd.iloc[:,1:].to_numpy()
x_train_full, x_valid_full, y_train_full, y_valid_full = train_test_split(x_data, y_data,test_size=0.2, random_state=42)
 if do_reshape:
x_train_full = np.reshape(x_train_full,(-1, 28, 28, 1))
x_valid_full = np.reshape(x_valid_full,(-1, 28, 28, 1))
x_test_full = np.reshape(x_test_full,(-1, 28, 28, 1))
return(x_train_full, y_train_full),(x_valid_full, y_valid_full), x_test_full
def show_imgs(x_train, y_train, rows=4):
classes = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
print(classes)
 for y, cls in enumerate(classes):
idxs = np.nonzero([i == y for i in y_train])
 idxs = np.random.choice(idxs[0], rows)
 for i , idx in enumerate(idxs):
plt_idx = i * len(classes)+ y + 1
plt.subplot(rows, len(classes), plt_idx)
 plt.imshow(x_train[idx].reshape(( 28, 28)))
 plt.axis("off")
 if i == 0:
plt.title(cls)
 plt.show()
train_text = train_df['text']
test_text = test_df['text']
if b_add_keyword:
for ii in range(train_text.shape[0]):
if not pd.isna(train_df.loc[ii, 'keyword']):
train_text[ii] = train_text[ii] + ' ' + train_df.loc[ii, 'keyword']
for ii in range(test_text.shape[0]):
if not pd.isna(test_df.loc[ii, 'keyword']):
test_text[ii] = test_text[ii] + ' ' + test_df.loc[ii, 'keyword']
for ii in cleaning:
train_text = train_text.apply(eval(ii))
test_text = test_text.apply(eval(ii)) 
<string_transform>
( x_train, y_train),(x_valid, y_valid), x_test = load_data(do_reshape=True)
 show_imgs(x_train, y_train, rows=4)
 print("Train shape", x_train.shape, y_train.shape)
 print("Valid shape", x_valid.shape, y_valid.shape)
 print("Test shape", x_test.shape )
all_words_train = []
for t in train_text.values:
all_words_train.extend(wordpunct_tokenize(t))
all_words_test = []
for t in test_text.values:
all_words_test.extend(wordpunct_tokenize(t))<train_model>
class average_CNN(tf.keras.Model):
def __init__(self, feature_maps = 32, kernel_sizes =(1,3,5), **kwargs):
super(average_CNN, self ).__init__(**kwargs)
 self.cnn_layers = []
for kernel_size in kernel_sizes:
self.cnn_layers.append(
 [Conv2D(feature_maps, kernel_size, activation="relu", kernel_initializer="he_uniform", padding="same"),
Conv2D(feature_maps, kernel_size, activation="relu", kernel_initializer="he_uniform", padding="same"),
BatchNormalization()
] 
)
 self.avg_layer = Average()
self.act = Activation('relu')
 self.bn4 = BatchNormalization()
self.pool = MaxPool2D()
self.drop = Dropout(0.2)
 def call(self, inputs):
x = inputs
self.middle_outs = []
for layer_set in self.cnn_layers:
for layer in layer_set:
x = layer(x)
 self.middle_outs.append(x)
 x = self.avg_layer(self.middle_outs)
 x = self.act(x)
 x = self.bn4(x)
 x = self.pool(x)
 x = self.drop(x)
 return x
class my_CNN(tf.keras.Model):
def __init__(self, **kwargs):
super(my_CNN, self ).__init__(**kwargs)
 self.avg_c1 = average_CNN(32,(1,3,5))
self.avg_c2 = average_CNN(64,(1,3,5))
self.avg_c3 = average_CNN(128,(1,3,5))
self.flat = Flatten()
self.den1 = Dense(128, activation="relu", kernel_initializer="he_uniform")
 self.drop = Dropout(0.2)
 self.den2 = Dense(10, activation="softmax")
 def call(self, inputs):
x = self.avg_c1(inputs)
 x = self.avg_c2(x)
 x = self.avg_c3(x)
 x = self.flat(x)
 x = self.den1(x)
 x = self.drop(x)
 x = self.den2(x)
 return x
counter = Counter(all_words_train)
 print('train
{}'.format(counter.most_common(10)))
 counter = Counter(all_words_test)
 print('test
{}'.format(counter.most_common(10)) )<string_transform>
def module_CNN() :
my_cnn = my_CNN()
my_cnn.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 return my_cnn
wordpunct_tokenize("this is great don't"), word_tokenize("this is great don't"), tokenizer.tokenize("this is great don't" )<categorify>
def model_CNN() :
model_input = keras.Input(shape=(28,28,1))
C1_1 = Conv2D(32, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(model_input)
 C1_2 = Conv2D(32, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(C1_1)
 C1_3 = BatchNormalization()(C1_2)
 C2_1 = Conv2D(32, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(model_input)
 C2_2 = Conv2D(32, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(C2_1)
 C2_3 = BatchNormalization()(C2_2)
 C3_1 = Conv2D(32, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(model_input)
 C3_2 = Conv2D(32, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(C3_1)
 C3_3 = BatchNormalization()(C3_2)
 avg1_1 = Average()([C1_3, C2_3, C3_3])
 avg1_2 = Activation('relu' )(avg1_1)
 avg1_3 = BatchNormalization()(avg1_2)
 mp1 = MaxPool2D()(avg1_3)
 out1 = Dropout(0.2 )(mp1)
 C1_1 = Conv2D(64, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(out1)
 C1_2 = Conv2D(64, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(C1_1)
 C1_3 = BatchNormalization()(C1_2)
 C2_1 = Conv2D(64, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(out1)
 C2_2 = Conv2D(64, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(C2_1)
 C2_3 = BatchNormalization()(C2_2)
 C3_1 = Conv2D(64, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(out1)
 C3_2 = Conv2D(64, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(C3_1)
 C3_3 = BatchNormalization()(C3_2)
 avg2_1 = Average()([C1_3, C2_3, C3_3])
 avg2_2 = Activation('relu' )(avg2_1)
 avg2_3 = BatchNormalization()(avg2_2)
 mp2 = MaxPool2D()(avg2_3)
 out2 = Dropout(0.2 )(mp2)
 C1_1 = Conv2D(128, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(out2)
 C1_2 = Conv2D(128, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(C1_1)
 C1_3 = BatchNormalization()(C1_2)
 C2_1 = Conv2D(128, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(out2)
 C2_2 = Conv2D(128, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(C2_1)
 C2_3 = BatchNormalization()(C2_2)
 C3_1 = Conv2D(128, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(out2)
 C3_2 = Conv2D(128, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(C3_1)
 C3_3 = BatchNormalization()(C3_2)
 avg3_1 = Average()([C1_3, C2_3, C3_3])
 avg3_2 = Activation('relu' )(avg3_1)
 avg3_3 = BatchNormalization()(avg3_2)
 mp3 = MaxPool2D()(avg3_3)
 out3 = Dropout(0.2 )(mp3)
 out = Flatten()(out3)
 out = Dense(128, activation="relu", kernel_initializer="he_uniform" )(out)
 out = Dropout(0.2 )(out)
 out = Dense(10, activation="softmax" )(out)
 model = tf.keras.Model(inputs=[model_input],outputs=[out])
 model.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 return model
train_text_enc = encode_text(train_text.values, tokenizer, max_len)
 test_text_enc = encode_text(test_text.values, tokenizer, max_len )<choose_model_class>
rescale_layer = tf.keras.Sequential([layers.experimental.preprocessing.Rescaling(1./255)])
 resize_and_rescale = tf.keras.Sequential([
layers.experimental.preprocessing.Rescaling(1./255)
 ])
 data_augmentation = tf.keras.Sequential([
layers.experimental.preprocessing.RandomContrast(0.5),
layers.experimental.preprocessing.RandomRotation(factor =(0.2), seed=42),
layers.experimental.preprocessing.RandomZoom(0.1),
])
 batch_size = 32
AUTOTUNE = -1
def prepare(ds, shuffle=False, augment=False):
x = resize_and_rescale(x)
 if shuffle:
ds = ds.shuffle(1000)
 ds = ds.batch(batch_size)
 if augment:
ds = data_augmentation(x, training=True)
 return ds.prefetch(-1 )
def build_bert_model(bert_layer, max_len):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 x = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 return model

<create_dataframe>
datagen = ImageDataGenerator(rescale=1/255.,
zoom_range = 0.3,
height_shift_range = 0.3,
width_shift_range = 0.3,
rotation_range = 25,
shear_range=0.5,
fill_mode="constant",
cval=0 
)
 valid_datagen = ImageDataGenerator(rescale=1./255)
 test_datagen = ImageDataGenerator(rescale=1./255 )
es = EarlyStopping(monitor='val_accuracy', mode='max', patience=3, verbose=1)
 m = build_bert_model(bert_layer, max_len)
 m.summary()
train_perf = pd.DataFrame()<choose_model_class>
start_time = time.time()
model, history = do_module_CNN(x_train, y_train, x_valid, y_valid)
 print("--- Model Train: %s seconds ---" %(time.time() - start_time))
summarize_diagnostics(history)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
m.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'] )<train_model>
start_time = time.time()
y_hat = model.predict_generator(test_datagen.flow(x_test, shuffle=False))
y_pred = np.argmax(y_hat,axis=1)
 print("--- Model Predict: %s seconds ---" %(time.time() - start_time))
print("y shape", y_pred.shape )
fit_hist = m.fit(train_text_enc, train_df['target'].values, epochs=10, validation_split=0.2, batch_size=32)
 train_perf = pd.concat(( train_perf, pd.DataFrame(fit_hist.history)) , ignore_index=True )<predict_on_test>
pd_submit=pd.DataFrame({"id": list(range(len(y_pred))),"label": y_pred})
 pd_submit.to_csv('submission.csv',index=False,header=True )
test_output = m.predict(test_text_enc)
 test_pred =(test_output >= 0.5 ).astype(int )<save_to_csv>
pd_submit['label'].value_counts()