Kaggle competition: Digit Recognizer
def map_example_to_dict(input_ids, attention_masks, token_type_ids, label):

 mapped_example = {
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": attention_masks,
}
return mapped_example, label
def encode_examples(texts_and_labels):

 input_ids_list = []
token_type_ids_list = []
attention_mask_list = []
label_list = []
for text, label in texts_and_labels:
bert_input = prepare_sequence(text)
 input_ids_list.append(bert_input['input_ids'])
 token_type_ids_list.append(bert_input['token_type_ids'])
 attention_mask_list.append(bert_input['attention_mask'])
 label_list.append([label])
 dataset = tf.data.Dataset.from_tensor_slices(
(input_ids_list, attention_mask_list, token_type_ids_list,
label_list)
)
 dataset_mapped = dataset.map(map_example_to_dict)
 return dataset_mapped<prepare_x_and_y>
train = pd.read_csv(".. /input/train.csv")
 print(train.shape)
 train.head()
X = train_data["text"]
y = train_data["target"]<split>
z_train = Counter(train['label'])
 z_train
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.10,
random_state=1 )<count_values>
test= pd.read_csv(".. /input/test.csv")
 print(test.shape)
 test.head()
n_training_examples = X_train.shape[0]
n_positive_training_examples = y_train.value_counts() [1]
n_negative_training_examples = y_train.value_counts() [0]
print(f'Number examples in training dataset: {n_training_examples}')
 print(f'Number of positive examples in training dataset: {n_positive_training_examples}')
 print(f'Number of negative examples in training dataset: {n_negative_training_examples}' )<define_variables>
x_train =(train.ix[:,1:].values ).astype('float32')
 y_train = train.ix[:,0].values.astype('int32')
 x_test = test.values.astype('float32' )
train_dataset = list(zip(X_train, y_train))
val_dataset = list(zip(X_val, y_val))<categorify>
x_train = x_train/255.0
x_test = x_test/255.0
ds_train_encoded = encode_examples(train_dataset ).shuffle(10000 ).batch(BATCH_SIZE)
 ds_val_encoded = encode_examples(val_dataset ).batch(BATCH_SIZE )<load_pretrained>
batch_size = 64
num_classes = 10
epochs = 20
input_shape =(28, 28, 1 )
def get_model() :
config = AutoConfig.from_pretrained(PRETRAINED_MODEL_NAME,
hidden_dropout_prob=0.2,
num_labels=LABELS_NUMBER)
 model = TFBertForSequenceClassification.from_pretrained(PRETRAINED_MODEL_NAME,
config=config)
 return model<choose_model_class>
y_train = keras.utils.to_categorical(y_train, num_classes)
 X_train, X_val, Y_train, Y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=42 )
model = get_model()
optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)
 loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')
 model.compile(optimizer=optimizer, loss=loss, metrics=[metric] )<train_model>
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',kernel_initializer='he_normal',input_shape=input_shape))
model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',kernel_initializer='he_normal'))
model.add(MaxPool2D(( 2, 2)))
 model.add(Dropout(0.20))
model.add(Conv2D(64,(3, 3), activation='relu',padding='same',kernel_initializer='he_normal'))
model.add(Conv2D(64,(3, 3), activation='relu',padding='same',kernel_initializer='he_normal'))
model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(128,(3, 3), activation='relu',padding='same',kernel_initializer='he_normal'))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.RMSprop() ,
metrics=['accuracy'])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.0001)
 datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=15,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False )
weight_for_0 =(1 / n_negative_training_examples)*(n_training_examples)/2.0
weight_for_1 =(1 / n_positive_training_examples)*(n_training_examples)/2.0
class_weight = {0: weight_for_0, 1: weight_for_1}
print('Weight for class 0: {:.2f}'.format(weight_for_0))
print('Weight for class 1: {:.2f}'.format(weight_for_1))<train_model>
datagen.fit(X_train)
 h = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 1, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction], )
model.fit(ds_train_encoded, epochs=EPOCHS_NUMBER, validation_data=ds_val_encoded,
class_weight = class_weight )<predict_on_test>
final_loss, final_acc = model.evaluate(X_val, Y_val, verbose=0)
 print("Final loss: {0:.6f}, final accuracy: {1:.6f}".format(final_loss, final_acc))
val_predictions = model.predict(ds_val_encoded)
 val_probabilities = softmax(val_predictions[0], axis=1)
 y_val_predictions = np.argmax(val_probabilities, axis=1 ).flatten()<categorify>
layer_outputs = [layer.output for layer in model.layers[:8]]
activation_model = models.Model(input=model.input, output=layer_outputs)
 activations = activation_model.predict(test_im.reshape(1,28,28,1))
first_layer_activation = activations[0]
plt.matshow(first_layer_activation[0, :, :, 4], cmap='viridis' )
def encode_test_examples(texts):

 input_ids_list = []
token_type_ids_list = []
attention_mask_list = []
for text in texts:
bert_input = prepare_sequence(text)
 input_ids_list.append(bert_input['input_ids'])
 token_type_ids_list.append(bert_input['token_type_ids'])
 attention_mask_list.append(bert_input['attention_mask'])
 dataset = tf.data.Dataset.from_tensor_slices(
(input_ids_list, attention_mask_list, token_type_ids_list)
)
 dataset_mapped = dataset.map(map_test_example_to_dict)
 return dataset_mapped
def map_test_example_to_dict(input_ids, attention_masks, token_type_ids):

 mapped_example = {
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": attention_masks,
}
return mapped_example<categorify>
model.layers[:-1]
X_test = test_data["text"]
test_dataset = list(X_test)
 ds_test_encoded = encode_test_examples(test_dataset ).batch(BATCH_SIZE )<predict_on_test>
layer_names = []
for layer in model.layers[:-1]:
layer_names.append(layer.name)
 images_per_row = 16
for layer_name, layer_activation in zip(layer_names, activations):
if layer_name.startswith('conv'):
n_features = layer_activation.shape[-1]
size = layer_activation.shape[1]
n_cols = n_features // images_per_row
display_grid = np.zeros(( size * n_cols, images_per_row * size))
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,:, :, col * images_per_row + row]
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255 ).astype('uint8')
 display_grid[col * size :(col + 1)* size,
row * size :(row + 1)* size] = channel_image
scale = 1./ size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
 plt.grid(False)
 plt.imshow(display_grid, aspect='auto', cmap='viridis' )
test_predictions = model.predict(ds_test_encoded)
 test_probabilities = softmax(test_predictions[0], axis=1)
 y_test_predictions = np.argmax(test_probabilities, axis=1 ).flatten()<save_to_csv>
layer_names = []
for layer in model.layers[:-1]:
layer_names.append(layer.name)
 images_per_row = 16
for layer_name, layer_activation in zip(layer_names, activations):
if layer_name.startswith('max'):
n_features = layer_activation.shape[-1]
size = layer_activation.shape[1]
n_cols = n_features // images_per_row
display_grid = np.zeros(( size * n_cols, images_per_row * size))
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,:, :, col * images_per_row + row]
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255 ).astype('uint8')
 display_grid[col * size :(col + 1)* size,
row * size :(row + 1)* size] = channel_image
scale = 1./ size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
 plt.grid(False)
 plt.imshow(display_grid, aspect='auto', cmap='viridis' )
final_submission = pd.DataFrame(data={"id":test_data["id"], "target":y_test_predictions})
 final_submission.to_csv("submissionTweets.csv", index=False )<set_options>
layer_names = []
for layer in model.layers[:-1]:
layer_names.append(layer.name)
 images_per_row = 16
for layer_name, layer_activation in zip(layer_names, activations):
if layer_name.startswith('drop'):
n_features = layer_activation.shape[-1]
size = layer_activation.shape[1]
n_cols = n_features // images_per_row
display_grid = np.zeros(( size * n_cols, images_per_row * size))
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,:, :, col * images_per_row + row]
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255 ).astype('uint8')
 display_grid[col * size :(col + 1)* size,
row * size :(row + 1)* size] = channel_image
scale = 1./ size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
 plt.grid(False)
 plt.imshow(display_grid, aspect='auto', cmap='viridis' )
warnings.filterwarnings('ignore')
<categorify>
Y_pred = model.predict(X_val)
 Y_pred_classes = np.argmax(Y_pred, axis = 1)
 Y_true_classes = np.argmax(Y_val, axis = 1 )
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 return np.array(all_tokens)
 def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_from_csv>
target_names = ["Class {}".format(i)for i in range(num_classes)]
print(classification_report(Y_true_classes, Y_pred_classes, target_names=target_names))
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<choose_model_class>
predicted_classes = model.predict_classes(X_test)
 submissions=pd.DataFrame({"ImageId": list(range(1,len(predicted_classes)+1)) ,
"Label": predicted_classes})
 submissions.to_csv("asd.csv", index=False, header=True )
<split><EOS> 
model.save('my_model_1.h5')
 json_string = model.to_json()