Kaggle competition: Digit Recognizer
print(torch.__version__ )<feature_engineering>
%matplotlib inline
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_URL(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_URL(x))
def remove_html(text):
html = re.compile(r'<.*?>')
 return html.sub(r'',text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_html(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_html(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x: remove_emoji(x))
test_df['text'] = test_df['text'].apply(lambda x: remove_emoji(x))
def remove_punct(text):
table = str.maketrans('','', string.punctuation)
 return text.translate(table)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_punct(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_punct(x))
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : correct_spellings(x))
test_df['text'] = test_df['text'].apply(lambda x : correct_spellings(x))<save_to_csv>
print("Tensorflow version " + tf.__version__)
 np.random.seed(42 )
test_df.to_csv("test.tsv", sep='\t', index=False, header=None)
 print(test_df.shape)
 train_val_df.to_csv("train_eval.tsv", sep='\t', index=False, header=None)
 print(train_val_df.shape)
<categorify>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
max_length = 100
def tokenizer_100(input_text):

 return tokenizer.encode(input_text, max_length=100, return_tensors='pt')[0]
TEXT = torchtext.data.Field(sequential=True, tokenize=tokenizer_100, use_vocab=False, lower=False,
include_lengths=True, batch_first=True, fix_length=max_length, pad_token=0)
 LABEL = torchtext.data.Field(sequential=False, use_vocab=False )<train_model>
train_data=pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_data=pd.read_csv(".. /input/digit-recognizer/test.csv" )
dataset_train_eval, dataset_test = torchtext.data.TabularDataset.splits(
 path='.', train='./train_eval.tsv', test='./test.tsv', format='tsv', fields=[('Text', TEXT),('Label', LABEL)] )<data_type_conversions>
Y_train=train_data['label']
X_train=train_data.drop('label', axis=1 )
print(tokenizer.convert_ids_to_tokens(item.Text.tolist()))
print(int(item.Label))<define_variables>
( x_train0, y_train0),(x_test0, y_test0)= mnist.load_data()
x_train1 = np.concatenate([x_train0, x_test0], axis=0)
 y_train1 = np.concatenate([y_train0, y_test0], axis=0)
 X_train_keras = x_train1.reshape(-1, 28*28)
 Y_train_keras = y_train1
batch_size = 32
dl_train = torchtext.data.Iterator(
 dataset_train, batch_size=batch_size, train=True)
 dl_eval = torchtext.data.Iterator(
 dataset_eval, batch_size=batch_size, train=False, sort=False)
 dl_test = torchtext.data.Iterator(
 dataset_test, batch_size=batch_size, train=False, sort=False)
 dataloaders_dict = {"train": dl_train, "val": dl_eval}<import_modules>
X_train = np.concatenate(( X_train.values, X_train_keras))
Y_train = np.concatenate(( Y_train, Y_train_keras))
print(transformers.__version__ )<load_pretrained>
unique, counts = np.unique(Y_train, return_counts=True)
 dict(zip(unique, counts))
model = BertModel.from_pretrained('bert-base-cased' )<set_options>
X_train = X_train.astype('float32')
 Y_train = Y_train.astype('float32')
 test_data=test_data.astype('float32')
 X_train = X_train / 255.0
test_data = test_data / 255.0
class BertForTwitter(nn.Module):
def __init__(self):
super(BertForTwitter, self ).__init__()
self.bert = model
self.cls = nn.Linear(in_features=768, out_features=9)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids):

 result = self.bert(input_ids)
 vec_0 = result[0]
vec_0 = vec_0[:, 0, :]
vec_0 = vec_0.view(-1, 768)
 output = self.cls(vec_0)
 return output<train_on_grid>
Y_train=to_categorical(Y_train, num_classes=10)
 print(f"Label size {Y_train.shape}" )
net = BertForTwitter()
net.train()
print('ネットワーク設定完了' )<categorify>
X_train, X_val, Y_train, Y_val=train_test_split(X_train, Y_train, test_size=0.3, random_state=42)
for param in net.parameters() :
param.requires_grad = False
for param in net.bert.encoder.layer[-1].parameters() :
param.requires_grad = True
for param in net.cls.parameters() :
param.requires_grad = True<choose_model_class>
model= Sequential()
model.add(Conv2D(input_shape=(28,28,1), filters=32, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.10))
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.10))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Conv2D(filters=128, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.2))
model.add(Conv2D(filters=256, kernel_size =(3,3), activation="relu"))
model.add(BatchNormalization())
 model.add(Conv2D(filters=512, kernel_size =(3,3), activation="relu"))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.3))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(10, activation='softmax'))
model.summary()
optimizer = optim.Adam([
{'params': net.bert.encoder.layer[-1].parameters() , 'lr': 5e-5},
{'params': net.cls.parameters() , 'lr': 1e-4}
])
 criterion = nn.CrossEntropyLoss() 
<train_model>
optimizer=Adam(lr=0.001,decay=0.0 )
def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):
max_acc = 0
Stop_flag = False
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("使用デバイス：", device)
 print('-----start-------')
 net.to(device)
 torch.backends.cudnn.benchmark = True
batch_size = dataloaders_dict["train"].batch_size
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
iteration = 1
for batch in(dataloaders_dict[phase]):
inputs = batch.Text[0].to(device)
 labels = batch.Label.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 loss = criterion(outputs, labels)
 _, preds = torch.max(outputs, 1)
 if phase == 'train':
loss.backward()
optimizer.step()
if(iteration % 50 == 0):
acc =(torch.sum(preds == labels.data)
 ).double() /batch_size
print('イテレーション {} || Loss: {:.4f} || 10iter.|| 本イテレーションの正解率：{}'.format(
 iteration, loss.item() , acc))
iteration += 1
epoch_loss += loss.item() * batch_size
epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_acc = epoch_corrects.double(
)/ len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,
phase, epoch_loss, epoch_acc))
if phase == "val":
if epoch_acc < max_acc:
count += 1
if count >= 3:
Stop_flag = True
else:
count = 0
max_acc = epoch_acc
print(count)
 if Stop_flag:
break
return net<train_model>
model.compile(optimizer= optimizer, loss='categorical_crossentropy', metrics=['accuracy'] )
num_epochs = 100
net_trained = train_model(net, dataloaders_dict,
criterion, optimizer, num_epochs=num_epochs )<load_from_csv>
learning_rate_redcuing=ReduceLROnPlateau(monitor='val_accuracy',
patience=5,
verbose=1,
factor=0.5,
min_lr=0.0001 )
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = ans_list
sample_submission<save_to_csv>
epochs = 30
batch_size = 32
sample_submission.to_csv("submission_plus.csv", index=False )<install_modules>
imagegen=ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 imagegen.fit(X_train )
!pip install nlpaug<load_pretrained>
history=model.fit_generator(imagegen.flow(X_train, Y_train,batch_size=batch_size),
epochs=epochs,
validation_data=(X_val, Y_val),
verbose=1,
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=[learning_rate_redcuing] )
!kaggle datasets download -d rtatman/glove-global-vectors-for-word-representation<install_modules>
model.save("MNIST_CNN_Model.h5")
 model.save_weights("MNIST_CNN_Model_weights.h5" )
!pip install nltk
!pip install gensim<load_pretrained>
results = model.predict(test_data)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<load_from_url><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("sample_submission.csv",index=False)
 submission.head()