Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_pretrained>
import numpy as np
import pandas as pd
from IPython.display import Image
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from tensorflow.python import keras
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, Conv2D, Dropout, MaxPool2D, BatchNormalization, Activation
from tensorflow.python.keras.callbacks import ReduceLROnPlateau
from tensorflow.python.keras.preprocessing.image import ImageDataGenerator
from tensorflow.python.keras.optimizers import RMSprop, Adam
from tensorflow.python.keras.activations import relu
from tensorflow.python.keras.utils import plot_model
from tensorflow.python.keras.losses import categorical_crossentropy
BERT_MODEL_NAME = 'bert-base-cased'
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_NAME )<create_dataframe>
digit_data = pd.read_csv('.. /input/train.csv')
 digit_data.head(5 )
train_dataset = ToxicCommentsDataset(
 train_df,
tokenizer,
max_token_len=MAX_TOKEN_COUNT 
)
 val_dataset = ToxicCommentsDataset(
 val_df,
tokenizer,
max_token_len=MAX_TOKEN_COUNT 
)
<create_dataframe>
img_rows, img_cols = 28,28
num_classes = 10
def data_prep_train(raw,val_frac):

 num_images = int(raw.shape[0])
 y_full = keras.utils.to_categorical(raw.label, num_classes)
 X_as_array = raw.values[:,1:]
X_shaped_array = X_as_array.reshape(num_images, img_rows, img_cols, 1)
 X_full = X_shaped_array / 255
X_train, X_val, y_train, y_val = train_test_split(X_full, y_full, test_size=val_frac)
 return X_train, X_val, y_train, y_val

 def data_prep_predict(raw):
num_images = int(raw.shape[0])
 X_as_array = raw.values
X_shaped_array = X_as_array.reshape(num_images, img_rows, img_cols, 1)
 X = X_shaped_array / 255
return X
test_dataset = ToxicCommentsDataset(
 test_df,
tokenizer,
max_token_len=MAX_TOKEN_COUNT,
test=True 
)
<load_pretrained>
def build_model(layer_sizes=[32, 32, 64, 64, 256], kernel_sizes=[5,5,3,3], activation = 'relu'):

 model = Sequential()
model.add(Conv2D(layer_sizes[0], kernel_size=kernel_sizes[0], padding = 'same', input_shape=(img_rows, img_cols, 1)))
 model.add(BatchNormalization())
 model.add(Activation(activation))
model.add(Conv2D(layer_sizes[1], kernel_size=kernel_sizes[1], padding = 'same'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(rate=0.25))
model.add(Conv2D(layer_sizes[2], kernel_size=kernel_sizes[2], padding = 'same'))
model.add(BatchNormalization())
 model.add(Activation(activation))
model.add(Conv2D(layer_sizes[3], kernel_size=kernel_sizes[3], padding = 'same'))
model.add(BatchNormalization())
 model.add(Activation(activation))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(rate=0.25))
model.add(Flatten())
 model.add(Dense(layer_sizes[4]))
model.add(BatchNormalization())
 model.add(Activation(activation))
model.add(Dropout(rate=0.5))
model.add(Dense(num_classes, activation='softmax'))
return model
my_model = build_model()
plot_model(my_model, to_file='my_model.png', show_shapes=True, show_layer_names=True)
 Image('my_model.png' )
train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 val_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)
 test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False )<set_options>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False, )
gc.collect()<choose_model_class>
def train_model(model, optimizer='adam', batch_size=64, epochs=1, verbose=1, callbacks=[]):

 model.compile(loss=categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])
 history = model.fit(datagen.flow(X_train, y_train, batch_size=batch_size),
epochs=epochs,
verbose=verbose,
validation_data=(X_val,y_val),
callbacks=callbacks)
 return history
class ToxicCommentTagger(nn.Module):
def __init__(self, n_classes: int, n_training_steps=None, n_warmup_steps=None):
super().__init__()
self.bert = BertModel.from_pretrained(BERT_MODEL_NAME, return_dict=True)
 self.classifier = nn.Linear(self.bert.config.hidden_size, n_classes)
 self.n_training_steps = n_training_steps
self.n_warmup_steps = n_warmup_steps
self.criterion = nn.BCELoss()
def forward(self, input_ids, attention_mask, labels=None):
output = self.bert(input_ids, attention_mask=attention_mask)
 output = self.classifier(output.pooler_output)
 output = torch.sigmoid(output)
 loss = 0
if labels is not None:
loss = self.criterion(output, labels)
 return loss, output<set_options>
X_train, X_val, y_train, y_val = data_prep_train(digit_data,0.1)
 leaky_relu = lambda x: relu(x, alpha=0.1)
 optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
 lr_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)
 my_model = build_model(activation=leaky_relu)
 history = train_model(my_model, optimizer=optimizer, epochs=40, batch_size = 128, verbose=1, callbacks=[lr_reduction])
 plt.figure(figsize=(20,9))
plt.plot(range(20,41),history.history['val_acc'][19:], color='red', label='validation accuracy')
 plt.plot(range(20,41),history.history['acc'][19:], color='blue', label='accuracy')
 legend = plt.legend(loc='best', shadow=True)
 plt.show()
<choose_model_class><EOS> 
subm_examples = pd.read_csv('.. /input/test.csv')
 X_subm = data_prep_predict(subm_examples)
 y_subm = my_model.predict(X_subm)
 n_rows = y_subm.shape[0]
y_subm = [np.argmax(y_subm[row,:])for row in range(n_rows)]
output = pd.DataFrame({'ImageId': range(1,n_rows+1), 'Label': y_subm})
 output.to_csv('submission.csv', index=False )