Kaggle competition: Kannada MNIST
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py 
<categorify>
train_df = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 test_df = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 dig_df = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv')
 print(train_df.shape, test_df.shape, dig_df.shape )
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
train_df['label'].value_counts().sort_index()
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
dig_df['label'].value_counts().sort_index()
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
X = train_df.drop(['label'], axis=1 ).values
Y = train_df['label']
X_test = test_df.drop(['id'], axis=1 ).values
X_dig = dig_df.drop(['label'], axis=1 ).values
Y_dig = dig_df['label']
print(X.shape, Y.shape, X_test.shape)
 print(X_dig.shape, Y_dig.shape )
train_input = bert_encode(train_df['text'].values, tokenizer, max_len=Max_length)
 test_input = bert_encode(test_df['text'].values, tokenizer, max_len=Max_length)
 train_labels = train_df['target_new'].values<choose_model_class>
X_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size=0.1 )
model = build_model(bert_layer, max_len=Max_length)
 model.summary()<train_model>
n_x = 28
train_img = X_train.reshape(( -1, n_x, n_x, 1)).astype('float32')/255.
 dev_img = X_dev.reshape(( -1, n_x, n_x, 1)).astype('float32')/255.
 test_img = X_test.reshape(( -1, n_x, n_x, 1)).astype('float32')/255.
 dig_img = X_dig.reshape(( -1, n_x, n_x, 1)).astype('float32')/255.
 print(train_img.shape, dev_img.shape, test_img.shape, dig_img.shape)
 train_labels = to_categorical(Y_train)
 dev_labels = to_categorical(Y_dev)
 dig_labels = to_categorical(Y_dig)
 print(train_labels.shape, dev_labels.shape, dig_labels.shape)
 print(Y_dig[8], dig_labels[8])
 plt.figure(figsize=(1,1))
plt.imshow(X_dig[8].reshape(( 28,28)) ,cmap=plt.cm.binary)
 plt.show()
checkpoint = ModelCheckpoint('model_BERT.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split = valid,
epochs = epochs_num,
callbacks=[checkpoint],
batch_size = batch_size_num
)<predict_on_test>
model = models.Sequential()
model.add(layers.Conv2D(32, kernel_size=3, padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(layers.Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(layers.BatchNormalization(momentum=0.15))
model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Conv2D(64, kernel_size=3, padding='same', activation='relu'))
model.add(layers.Conv2D(64, kernel_size=3, padding='same', activation='relu'))
model.add(layers.BatchNormalization(momentum=0.15))
model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Conv2D(64, kernel_size=5, padding='same', activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Conv2D(128, kernel_size=3, padding='same', activation='relu'))
model.add(layers.Conv2D(128, kernel_size=3, padding='same', activation='relu'))
model.add(layers.BatchNormalization(momentum=0.15))
model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Flatten())
 model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(10, activation='softmax'))
model.summary()
model.load_weights('model_BERT.h5')
 test_pred = model.predict(test_input)
 test_pred_int = test_pred.round().astype('int')
 train_pred = model.predict(train_input)
 train_pred_int = train_pred.round().astype('int' )<create_dataframe>
es = EarlyStopping(monitor='val_loss', verbose=2, patience=10)
 initial_learningrate=2e-3
model.compile(optimizer=RMSprop(lr=initial_learningrate), loss='categorical_crossentropy', metrics=['accuracy'] )
plt.style.use('ggplot')
 pred = pd.DataFrame(test_pred, columns=['preds'])
 pred.plot.hist()<save_to_csv>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',patience=10,factor=0.6,min_lr=0.00001, verbose=2)
 def lr_decay(epoch):
return initial_learningrate * 0.99 ** epoch
submission['target'] = test_pred_int
submission.to_csv("submission_final.csv", index=False)
 submission.head(5 )<categorify>
epochs = 50
batch_size = 64
history = model.fit_generator(data_augment.flow(train_img, train_labels, batch_size=batch_size),
epochs=epochs, steps_per_epoch=train_img.shape[0]//batch_size,
validation_data=(dev_img, dev_labels), callbacks=[learning_rate_reduction,LearningRateScheduler(lr_decay),es,checkpoint])
 model.load_weights('BWeight.md5' )
def encode(text, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
text = tokenizer.tokenize(str(text))
text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<predict_on_test>
model.evaluate(dig_img, dig_labels )
ResultDict={1:'Disaster', 0:'Non Disaster'}
def predict_review(input_text):
input_seq = encode(input_text, tokenizer, max_len=Max_length)
 predict_result = model.predict(input_seq)
 i = predict_result[0][0].round().astype('int')
 print('Input:', input_text)
 pre_score = round(float(predict_result[0][0])*100, 4)
 print(f'Output: {ResultDict[i]}({pre_score}%)
 ' )<define_variables>
pred_dig = model.predict(dig_img)
 dig_df['pred'] = np.argmax(pred_dig, axis=1 )
userInput1 =
userInput2 = How did I know as soon as I walked out of class that Calgary would flood again today for sixth year in a row premium costs for windstorm insurance to climb.this time by 5 percent Truth...
 userInput6 =
userInput7 = Who is bringing the tornadoes and floods.Who is bringing the climate change.
 userInput9 =
userInput10 =
userInput11 =
userInput12 =
userInput13 =
userInput14 =<predict_on_test>
dig_df['correct'] = dig_df['label'] - dig_df['pred']
errors = dig_df[dig_df['correct'] != 0]
error_list = errors.index
print('Number of errors is ', len(errors))
print('The indices are ', error_list )
predict_review(userInput1)
 predict_review(userInput2)
 predict_review(userInput3)
 predict_review(userInput4)
 predict_review(userInput5)
 predict_review(userInput6)
 predict_review(userInput7 )<predict_on_test>
predictions = model.predict(test_img)
 print(predictions.shape )
predict_review(userInput8)
 predict_review(userInput9)
 predict_review(userInput10)
 predict_review(userInput11)
 predict_review(userInput12)
 predict_review(userInput13)
 predict_review(userInput14 )<install_modules>
predicted_labels = np.argmax(predictions, axis=1 )
!pip install fairseq
!pip install fastBPE
!pip install transformers<categorify>
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 submission['label'] = predicted_labels
submission.to_csv('submission.csv', index=False )