Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
tf.__version__
if torch.cuda.is_available() :
device = torch.device("cuda")
 else:
device = torch.device("cpu")
 device<count_duplicates>
train = pd.read_csv(r'/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv(r'/kaggle/input/digit-recognizer/test.csv')
 train.shape, test.shape
dupli_sum = train.duplicated().sum()
if(dupli_sum>0):
print(dupli_sum, " duplicates found
removing...")
 train = train.loc[False==train.duplicated() , :]
else:
print("no duplicates found")
 train<prepare_x_and_y>
X_train = x_train = train.drop(['label'],1)
 Y_train = train['label']
x_test = test
X_train = train["text"].values
y_train = train["target"].values<load_pretrained>
X_train = X_train.astype('float32')
 x_test = x_test.astype('float32')
 X_train = X_train/255
x_test - x_test/255
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
 lens = []
for text in X_train:
encoded_dict = tokenizer.encode_plus(text, add_special_tokens=True, return_tensors='pt')
 lens.append(encoded_dict['input_ids'].size() [1] )<categorify>
Y_train= tf.keras.utils.to_categorical(Y_train, 10)
 Y_train.shape
sequence_length = 58
X_train_tokens = []
for text in X_train:
encoded_dict = tokenizer.encode_plus(text,
add_special_tokens=True,
max_length=sequence_length,
padding="max_length",
return_tensors='pt',
truncation=True)
 X_train_tokens.append(encoded_dict['input_ids'] )<concatenate>
x_train, val_x, y_train, val_y = train_test_split(X_train, Y_train, test_size=0.20 )
X_train_tokens = torch.cat(X_train_tokens, dim=0)
 y_train = torch.tensor(y_train )<train_model>
es = EarlyStopping(monitor='loss', patience=12)
 filepath="/kaggle/working/bestmodel.h5"
md = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min' )
print('Original:
', X_train[5])
 print('Tokenization:
', X_train_tokens[5] )<split>
datagen = ImageDataGenerator(zoom_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1,
rotation_range = 10 )
batch_size = 32
dataset = TensorDataset(X_train_tokens, y_train.float())
 train_size = int(0.80 * len(dataset))
val_size = len(dataset)- train_size
train_set, val_set = random_split(dataset, [train_size, val_size])
 train_dataloader = DataLoader(train_set,
sampler=RandomSampler(train_set),
batch_size=batch_size)
 validation_dataloader = DataLoader(val_set,
sampler=RandomSampler(val_set),
batch_size=batch_size )<categorify>
epochs = 30
num_classes = 10
batch_size = 30
input_shape =(28, 28, 1)
 adam = tf.keras.optimizers.Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.999, amsgrad=False )
bert = BertModel.from_pretrained("bert-base-uncased")
 bert.to(device)
 for batch in train_dataloader:
batch_features = batch[0].to(device)
 bert_output = bert(input_ids=batch_features)
 print("bert output: ", type(bert_output), len(bert_output))
print("first entry: ", type(bert_output[0]), bert_output[0].size())
 print("second entry: ", type(bert_output[1]), bert_output[1].size())
 break<choose_model_class>
model = Sequential()
model.add(Conv2D(32,(3, 3), padding='same', input_shape=input_shape, activation= tf.nn.relu))
model.add(Conv2D(32,(3, 3), padding='same', activation= tf.nn.relu))
model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64,(3, 3), padding='same', activation= tf.nn.relu))
model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))

 model.add(Flatten())
 model.add(Dense(1024, activation=tf.nn.relu))
model.add(Dropout(0.25))

 model.add(Dense(512, activation=tf.nn.relu))
model.add(Dropout(0.25))
model.add(Dense(256, activation=tf.nn.relu))
model.add(Dropout(0.5))
model.add(Dense(10, activation= tf.nn.softmax))
model.compile(optimizer= adam, loss= tf.keras.losses.categorical_crossentropy, metrics=["accuracy"])
 model.summary()
class BertClassifier(nn.Module):
def __init__(self):
super(BertClassifier, self ).__init__()
self.bert = BertModel.from_pretrained('bert-base-uncased')
 self.linear = nn.Linear(768, 1)
 self.sigmoid = nn.Sigmoid()
def forward(self, tokens):
bert_output = self.bert(input_ids=tokens)
 linear_output = self.linear(bert_output[1])
 proba = self.sigmoid(linear_output)
 return proba<compute_test_metric>
History = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
epochs = epochs,
validation_data =(val_x, val_y),
callbacks = [es,md],
shuffle= True 
)
def eval(y_batch, probas):
preds_batch_np = np.round(probas.cpu().detach().numpy())
 y_batch_np = y_batch.cpu().detach().numpy()
acc = accuracy_score(y_true=y_batch_np, y_pred=preds_batch_np)
 f1 = f1_score(y_true=y_batch_np, y_pred=preds_batch_np, average='weighted')
 return acc, f1 
<train_model>
model1 = load_model("/kaggle/working/bestmodel.h5" )
def train(model, optimizer, scheduler, epochs, name):
history = []
best_f1 = 0
model.train()
for epoch in range(epochs):
print("=== Epoch: ", epoch+1, " / ", epochs, " ===")
 acc_total = 0
f1_total = 0
for it, batch in enumerate(train_dataloader):
x_batch, y_batch = [batch[0].to(device), batch[1].to(device)]
probas = torch.flatten(model(tokens=x_batch))
acc_f1_batch = eval(y_batch, probas)
 acc_total, f1_total = acc_total + acc_f1_batch[0], f1_total + acc_f1_batch[1]
model.zero_grad()
loss_func = nn.BCELoss()
batch_loss = loss_func(probas, y_batch)
 batch_loss.backward()
optimizer.step()
scheduler.step()
acc_total = acc_total/len(train_dataloader)
 f1_total = f1_total/len(train_dataloader)
 print("accuracy: ", acc_total, "
f1: ", f1_total)
 acc_val_total = 0
f1_val_total = 0
for batch in validation_dataloader:
x_batch, y_batch = [batch[0].to(device), batch[1].to(device)]
with torch.no_grad() :
probas = torch.flatten(model(tokens=x_batch))
acc_f1_val_batch = eval(y_batch, probas)
 acc_val_total, f1_val_total = acc_val_total + acc_f1_val_batch[0], f1_val_total + acc_f1_val_batch[1]
acc_val_total = acc_val_total/len(validation_dataloader)
 f1_val_total = f1_val_total/len(validation_dataloader)
 print("validation accuracy: ", acc_val_total, "
validation f1: ", f1_val_total, "
")
 if(f1_val_total>best_f1):
torch.save(model, name+".pt")
 best_f1 = f1_val_total
history.append({"acc":acc_total, "f1":f1_total, "acc_val":acc_val_total, "f1_val":f1_val_total})
 return [torch.load(name+".pt"), history]<normalization>
pred = model1.predict(x_test)
 pred_class = model1.predict_classes(x_test )
<train_model><EOS> 
submissions=pd.DataFrame({"ImageId": list(range(1,len(pred_class)+1)) ,
"Label": pred_class})
 submissions.to_csv("submissions.csv", index=False, header=True)
 submissions