Kaggle competition: Digit Recognizer
def tokenize_txts(txts, max_len = 40):
res = tokenizer(
 text = [tokenizer.tokenize(txt)for txt in txts],
max_length = max_len,
padding = 'max_length',
truncation = True,
is_split_into_words = True, 
)
 return {
'input_word_ids': res['input_ids'],
'input_mask': res['attention_mask'],
'input_type_ids': res['token_type_ids'],
}
def detokenize_txt(token):
res = tokenizer.decode(token)
 return res<define_variables>
%matplotlib inline
MAX_LEN = 35
BATCH_SIZE = 32<string_transform>
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )
train_tokens = tokenize_txts(train_txts, MAX_LEN)
 val_tokens = tokenize_txts(val_txts, MAX_LEN)
 test_tokens = tokenize_txts(test_txts, MAX_LEN)
<string_transform>
y_train = train["label"].astype('float32')
 x_train = train.drop(labels = ["label"],axis = 1 ).astype('float32')
 y_train.value_counts()
print('Orginal txt: ', train_txts[0])
 print()
sample = train_tokens['input_word_ids'][0]
print('Tokenized txt:', sample)
 print()
print('Detokenizd txt:', detokenize_txt(sample))<define_variables>
x_train = x_train / 255.0
test = test / 255.0
train_ds = tf.data.Dataset.from_tensor_slices(( train_tokens, y_train))
val_ds = tf.data.Dataset.from_tensor_slices(( val_tokens, y_val))
test_ds = tf.data.Dataset.from_tensor_slices(( test_tokens, y_test))
train_ds = train_ds.batch(BATCH_SIZE)
 val_ds = val_ds.batch(BATCH_SIZE)
 test_ds = test_ds.batch(BATCH_SIZE )<compute_train_metric>
y_train = keras.utils.to_categorical(y_train, 10 )
class MyF1(tf.keras.metrics.Metric):
def __init__(self, name = 'mf1_score'):
super(MyF1, self ).__init__(name)
 self.p = tf.metrics.Precision()
self.r = tf.metrics.Recall()
self.f1 = self.add_weight(name="f1", initializer="zeros")
 def update_state(self, actual, predicted, sample_weight = None):
self.p.update_state(actual, predicted)
 self.r.update_state(actual, predicted)
 self.f1.assign(2 * self.p.result() * self.r.result() /(self.p.result() + self.r.result()))
def reset_states(self,):
self.p.reset_states()
self.r.reset_states()
self.f1.assign(0.0)
 def result(self,):
return self.f1<init_hyperparams>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(512, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
class CSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, lr, freeze_epoch ,batch_size, data_size):
super(CSchedule, self ).__init__()
self.lr = lr
self.bs = batch_size
self.ds = data_size
self.freeze_epoch = freeze_epoch
def __call__(self, step):
epoch = step /(self.ds / self.bs)+ 1
if not self.freeze_epoch or epoch < self.freeze_epoch:
return self.lr /(tf.cast(epoch, tf.float32)+1)
 else:
return self.lr * 10 /(tf.cast(epoch, tf.float32)+1)
 clr = CSchedule(2e-5, 3, BATCH_SIZE, len(train_txts))
plt.plot([clr(x)for x in tf.range(1, 1457, dtype=tf.float32)])
 plt.xlabel('steps')
 plt.ylabel('learning rate')
 plt.title('Learning Rate Schedule')
 plt.show()<define_variables>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )
bert_handler = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3'<choose_model_class>
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )
class MClassifier(tf.keras.Model):
def __init__(self, dropout_rate):
super(MClassifier, self ).__init__()
self.bert_layer = hub.KerasLayer(
 bert_handler,
name = 'feature_ext',
trainable = True, 
)
 self.dropout = tf.keras.layers.Dropout(dropout_rate)
 self.proba = tf.keras.layers.Dense(1, activation = 'sigmoid')
 def call(self, X_in, training):
x = self.bert_layer(X_in, training = training)
 x = x['pooled_output']
x = self.dropout(x, training = training)
 output = self.proba(x)
 return output<choose_model_class>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(x_train )
LEARNING_RATE = 2e-5
EPOCHS = 5
DP_RATE = 0.3
loss_objective = tf.keras.losses.BinaryCrossentropy()<compute_test_metric>
model.fit_generator(datagen.flow(x_train,y_train, batch_size = 86),epochs = 30, steps_per_epoch = x_train.shape[0]//64)
def model_evaluation(model, ds, name):
acc = tf.keras.metrics.BinaryAccuracy()
f1 = MyF1()
total_loss = []
y_hats = []
for X, y in ds:
y_hat = model(X, training = False)
 loss = loss_objective(y, y_hat)
 acc.update_state(y, y_hat)
 f1.update_state(y, y_hat)
 total_loss.append(loss.numpy())
 y_hats.append(y_hat)
 y_hats = tf.concat(y_hats, axis = 0 ).numpy()
print(f'{name}, loss {np.mean(total_loss):.3f}, acc {acc.result().numpy() :.3f}, f1 {f1.result().numpy() :.3f}')
 return y_hats, acc.result().numpy() , f1.result().numpy()<train_on_grid>
pred = model.predict(test)
 final_pred = np.argmax(pred,axis = 1 )
@tf.function
def train_step(model, tr_vars, X, y):
with tf.GradientTape() as tape:
y_hat = model(X, training = True)
 loss = loss_objective(y, y_hat)
 grads = tape.gradient(loss, tr_vars)
 return loss, grads, y_hat
def train_model(model, epochs, freeze_bert_on_epoch = None):
clr = CSchedule(LEARNING_RATE, freeze_bert_on_epoch, BATCH_SIZE, len(train_txts))
optimizer = tf.keras.optimizers.Adam(clr)
 acc = tf.keras.metrics.BinaryAccuracy()
f1 = MyF1()
for epoch in range(1, epochs + 1):
if freeze_bert_on_epoch is not None and epoch == freeze_bert_on_epoch:
model.get_layer('feature_ext' ).trainable = False
acc.reset_states()
f1.reset_states()
total_loss = []
for X, y in train_ds:
tr_vars = model.trainable_variables
loss, grads, y_hat = train_step(model, tr_vars, X, y)
 optimizer.apply_gradients(zip(grads, tr_vars))
acc.update_state(y, y_hat)
 f1.update_state(y, y_hat)
 total_loss.append(loss.numpy())
 print(f'epoch {epoch}, loss {np.mean(total_loss):.3f}, acc {acc.result().numpy() :.3f}, f1 {f1.result().numpy() :.3f}')
 model_evaluation(model, val_ds, 'val')
 print()
y_test_hat, _, _ = model_evaluation(model, test_ds, 'test')
 return y_test_hat<prepare_x_and_y>
sample_sub = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv' )
<compute_test_metric><EOS> 
sample_sub = pd.DataFrame({"ImageId": list(range(1,len(final_pred)+1)) ,
"Label": final_pred})
 sample_sub.to_csv('CNN_submission1.csv', index=False)
 sample_sub.head()