Kaggle competition: Digit Recognizer
test_df.to_csv("test.tsv", sep='\t', index=False, header=None)
 print(test_df.shape)
 train_val_df.to_csv("train_eval.tsv", sep='\t', index=False, header=None)
 print(train_val_df.shape )<categorify>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
max_length = 50
def tokenizer_50(input_text):

 return tokenizer.encode(input_text, max_length=50, return_tensors='pt')[0]
TEXT = torchtext.data.Field(sequential=True, tokenize=tokenizer_50, use_vocab=False, lower=False,
include_lengths=True, batch_first=True, fix_length=max_length, pad_token=0)
 LABEL = torchtext.data.Field(sequential=False, use_vocab=False )<load_pretrained>
test_data=pd.read_csv('.. /input/digit-recognizer/test.csv')
 train_data=pd.read_csv('.. /input/digit-recognizer/train.csv' )
tokenizer = BertTokenizer.from_pretrained('bert-base-cased' )<load_from_csv>
training=np.array(train_data,dtype='float32')
 testing=np.array(test_data,dtype='float32' )
dataset_train_eval, dataset_test = torchtext.data.TabularDataset.splits(path='.', train='./train_eval.tsv', test='./test.tsv', format='tsv', fields=[('Text', TEXT),('Label', LABEL)] )<split>
x_train=training[:,1:]/255
y_train=training[:,0]
dataset_train, dataset_eval = dataset_train_eval.split(
 split_ratio=1.0 - 1800/7613, random_state=random.seed(1234))
print(dataset_train.__len__())
 print(dataset_eval.__len__())
 print(dataset_test.__len__() )<data_type_conversions>
x_test=testing[:,0:]/255
print(tokenizer.convert_ids_to_tokens(item.Text.tolist()))
print(int(item.Label))<define_variables>
y_train_cat=to_categorical(y_train )
batch_size = 32
dl_train = torchtext.data.Iterator(
 dataset_train, batch_size=batch_size, train=True)
 dl_eval = torchtext.data.Iterator(
 dataset_eval, batch_size=batch_size, train=False, sort=False)
 dl_test = torchtext.data.Iterator(
 dataset_test, batch_size=batch_size, train=False, sort=False)
 dataloaders_dict = {"train": dl_train, "val": dl_eval}<load_pretrained>
model =Sequential()
model.add(Conv2D(64,kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1)))
 model.add(Conv2D(64,kernel_size=(3, 3),activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 128, kernel_size =(3,3),activation ='relu'))
model.add(Conv2D(filters = 128, kernel_size =(3,3),activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3),activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(BatchNormalization())
 model.add(Dense(512, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model = BertModel.from_pretrained('bert-base-cased' )<set_options>
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.RMSprop() ,
metrics=['accuracy'])
 learning_rate_reduction = ReduceLROnPlateau(monitor='accuracy',
patience=3,
verbose=1,
factor=0.3,
min_lr=0.0001 )
class BertForTwitter(nn.Module):
def __init__(self):
super(BertForTwitter, self ).__init__()
self.bert = model
self.cls = nn.Linear(in_features=768, out_features=2)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids):

 result = self.bert(input_ids)
 vec_0 = result[0]
vec_0 = vec_0[:, 0, :]
vec_0 = vec_0.view(-1, 768)
 output = self.cls(vec_0)
 return output<train_on_grid>
epochs_range = 60
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
 model.fit(X_train,y_train_cat,batch_size=256,epochs=epochs_range )
net = BertForTwitter()
net.train()
print('ネットワーク設定完了' )<categorify>
y_pred=model.predict_classes(X_test )
for param in net.parameters() :
param.requires_grad = False
for param in net.bert.encoder.layer[-1].parameters() :
param.requires_grad = True
for param in net.cls.parameters() :
param.requires_grad = True<choose_model_class>
sample=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv' )
optimizer = optim.Adam([
{'params': net.bert.encoder.layer[-1].parameters() , 'lr': 5e-5},
{'params': net.cls.parameters() , 'lr': 1e-4}
])
 criterion = nn.CrossEntropyLoss() 
<train_model>
temp=pd.DataFrame({'ImageId':sample['ImageId'],'Label':y_pred} )
def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):
max_acc = 0
Stop_flag = False
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("使用デバイス：", device)
 print('-----start-------')
 net.to(device)
 torch.backends.cudnn.benchmark = True
batch_size = dataloaders_dict["train"].batch_size
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
iteration = 1
for batch in(dataloaders_dict[phase]):
inputs = batch.Text[0].to(device)
 labels = batch.Label.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 loss = criterion(outputs, labels)
 _, preds = torch.max(outputs, 1)
 if phase == 'train':
loss.backward()
optimizer.step()
if(iteration % 50 == 0):
acc =(torch.sum(preds == labels.data)
 ).double() /batch_size
print('イテレーション {} || Loss: {:.4f} || 10iter.|| 本イテレーションの正解率：{}'.format(
 iteration, loss.item() , acc))
iteration += 1
epoch_loss += loss.item() * batch_size
epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_acc = epoch_corrects.double(
)/ len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,
phase, epoch_loss, epoch_acc))
if phase == "val":
if epoch_acc < max_acc:
count += 1
if count >= 3:
Stop_flag = True
else:
count = 0
max_acc = epoch_acc
print(count)
 if Stop_flag:
break
return net<train_model>
csv_data=temp.to_csv('pred.csv',index=False )