Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_from_csv>
sns.set()
dataset_train_eval, dataset_test = torchtext.data.TabularDataset.splits(path='.', train='./train_eval.tsv', test='./test.tsv', format='tsv', fields=[('Text', TEXT),('Label', LABEL)] )<split>
digits_PATH = '/kaggle/input/digit-recognizer/'
digits_train = pd.read_csv(digits_PATH+'train.csv' )
dataset_train, dataset_eval = dataset_train_eval.split(
 split_ratio=1.0 - 1800/7613, random_state=random.seed(1234))
print(dataset_train.__len__())
 print(dataset_eval.__len__())
 print(dataset_test.__len__() )<data_type_conversions>
X, y = digits_train.iloc[:,1:].values/255, digits_train.iloc[:,0].values
X = X.reshape(-1,28, 28, 1 )
print(tokenizer.convert_ids_to_tokens(item.Text.tolist()))
print(int(item.Label))<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9, random_state=0 )
batch_size = 32
dl_train = torchtext.data.Iterator(
 dataset_train, batch_size=batch_size, train=True)
 dl_eval = torchtext.data.Iterator(
 dataset_eval, batch_size=batch_size, train=False, sort=False)
 dl_test = torchtext.data.Iterator(
 dataset_test, batch_size=batch_size, train=False, sort=False)
 dataloaders_dict = {"train": dl_train, "val": dl_eval}<load_pretrained>
class MultiCNN() :
def __init__(self, model_generator, num_models=1):
self.models = []
self.create_models(model_generator, num_models)
 def create_models(self, model_generator, num_models=1):
for i in range(0,num_models):
m = keras.models.Sequential(model_generator())
 m.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 self.models.append(m)
 def fit(self, train_inputs, train_targets, early_stop=True, verbose=2, batch_size=200, epochs=100):
callback = []
if early_stop:
callback.append(keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-6, patience=5))
for m in self.models:
X_train2, X_val2, y_train2, y_val2 = train_test_split(train_inputs, train_targets, test_size = 0.2)
 m.fit(X_train2, y_train2,
batch_size= batch_size,
epochs = epochs,
callbacks=callback,
validation_data =(X_val2, y_val2),
verbose=verbose 
)
 def fit_generator(self, generator, train_inputs, train_targets, early_stop=True, verbose=2, batch_size=32, epochs=100):
callback = []
if early_stop:
callback.append(keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-6, patience=5))
for m in self.models:
X_train2, X_val2, y_train2, y_val2 = train_test_split(train_inputs, train_targets, test_size = 0.2)
 m.fit_generator(generator.flow(X_train2,y_train2, batch_size=batch_size),
epochs = epochs,
callbacks=callback,
validation_data =(X_val2, y_val2),
verbose=verbose 
)
 def predict(self, test_inputs):
prediction = [model.predict(test_inputs)for model in self.models]
return sum(prediction)/len(prediction)
 def acuracia(self, test_input, test_target):
prediction = self.predict(test_input)
 prediction = np.argmax(prediction, axis=1)
 return accuracy_score(prediction, test_target)
 def confusion_matrix(self, test_input, test_target):
y_pred = self.predict(test_input)
 y_pred = np.argmax(y_pred, axis=1)
 confusao = confusion_matrix(y_pred, test_target)
 fig, ax = plt.subplots(1, figsize=(10,10))
sns.heatmap(pd.DataFrame(confusao), ax=ax, cbar=False, annot=True)
 ax.set_title('Matriz de confusão', size=20)
 ax.set_yticklabels(ax.get_xticklabels() , rotation=0, size=15)
 ax.set_xticklabels(ax.get_yticklabels() , rotation=0, size=15)
 plt.show()
def acuracia_individual(self, test_input, test_target):
return [accuracy_score(np.argmax(m.predict(X_test), axis=1), y_test)for m in self.models]
model = BertModel.from_pretrained('bert-base-cased' )<set_options>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_default = MultiCNN(make_CNN, num_models=15)
 multiCNN_default.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_default.acuracia(X_test, y_test )
class BertForTwitter(nn.Module):
def __init__(self):
super(BertForTwitter, self ).__init__()
self.bert = model
self.cls = nn.Linear(in_features=768, out_features=2)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids):

 result = self.bert(input_ids)
 vec_0 = result[0]
vec_0 = vec_0[:, 0, :]
vec_0 = vec_0.view(-1, 768)
 output = self.cls(vec_0)
 return output<train_on_grid>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_plus_layer = MultiCNN(make_CNN, num_models=15)
 multiCNN_plus_layer.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_plus_layer.acuracia(X_test, y_test )
net = BertForTwitter()
net.train()
print('ネットワーク設定完了' )<categorify>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.AveragePooling2D(pool_size =(2,2), strides = 2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.AveragePooling2D(pool_size =(2,2), strides = 2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_pooling = MultiCNN(make_CNN, num_models=15)
 multiCNN_pooling.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_pooling.acuracia(X_test, y_test )
for param in net.parameters() :
param.requires_grad = False
for param in net.bert.encoder.layer[-1].parameters() :
param.requires_grad = True
for param in net.cls.parameters() :
param.requires_grad = True<choose_model_class>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_batch_normalization = MultiCNN(make_CNN, num_models=15)
 multiCNN_batch_normalization.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_batch_normalization.acuracia(X_test, y_test )
optimizer = optim.Adam([
{'params': net.bert.encoder.layer[-1].parameters() , 'lr': 5e-5},
{'params': net.cls.parameters() , 'lr': 1e-4}
])
 criterion = nn.CrossEntropyLoss() 
<train_model>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_dropout = MultiCNN(make_CNN, num_models=15)
 multiCNN_dropout.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_dropout.acuracia(X_test, y_test )
def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):
max_acc = 0
Stop_flag = False
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("使用デバイス：", device)
 print('-----start-------')
 net.to(device)
 torch.backends.cudnn.benchmark = True
batch_size = dataloaders_dict["train"].batch_size
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
iteration = 1
for batch in(dataloaders_dict[phase]):
inputs = batch.Text[0].to(device)
 labels = batch.Label.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 loss = criterion(outputs, labels)
 _, preds = torch.max(outputs, 1)
 if phase == 'train':
loss.backward()
optimizer.step()
if(iteration % 50 == 0):
acc =(torch.sum(preds == labels.data)
 ).double() /batch_size
print('イテレーション {} || Loss: {:.4f} || 10iter.|| 本イテレーションの正解率：{}'.format(
 iteration, loss.item() , acc))
iteration += 1
epoch_loss += loss.item() * batch_size
epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_acc = epoch_corrects.double(
)/ len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,
phase, epoch_loss, epoch_acc))
if phase == "val":
if epoch_acc < max_acc:
count += 1
if count >= 3:
Stop_flag = True
else:
count = 0
max_acc = epoch_acc
print(count)
 if Stop_flag:
break
return net<train_model>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_200 = MultiCNN(make_CNN, num_models=15)
 multiCNN_200.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_200.acuracia(X_test, y_test )
num_epochs = 50
net_trained = train_model(net, dataloaders_dict,
criterion, optimizer, num_epochs=num_epochs )<load_from_csv>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_bigger_kernel = MultiCNN(make_CNN, num_models=15)
 multiCNN_bigger_kernel.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_bigger_kernel.acuracia(X_test, y_test )
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = ans_list
sample_submission<save_to_csv>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_min_epochs = MultiCNN(make_CNN, num_models=15)
 multiCNN_min_epochs.fit(X_train, y_train, verbose = 0, early_stop=True, epochs=10)
 multiCNN_min_epochs.acuracia(X_test, y_test )
sample_submission.to_csv("submission_plus.csv", index=False )<import_modules>
datagen = keras.preprocessing.image.ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
zca_epsilon=1e-06,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
brightness_range=None,
zoom_range=0.1,
fill_mode="nearest",
cval=0.0,
horizontal_flip=False,
vertical_flip=False,
rescale=None,
preprocessing_function=None,
data_format=None,
validation_split=0.0,
dtype=None, 
)
 datagen.fit(X_train )
import torch
import pandas as pd
import numpy as np
from transformers import AutoModelForSequenceClassification, AutoTokenizer<string_transform>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_data_augmentation = MultiCNN(make_CNN, num_models=15)
 multiCNN_data_augmentation.fit_generator(datagen, X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_data_augmentation.acuracia(X_test, y_test )
class Dataset:
def __init__(self, text, tokenizer, max_len):
self.text = text
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.text)
 def __getitem__(self, item):
text = str(self.text[item])
 inputs = self.tokenizer(
 text,
max_length=self.max_len,
padding="max_length",
truncation=True 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
return {
"input_ids": torch.tensor(ids, dtype=torch.long),
"attention_mask": torch.tensor(mask, dtype=torch.long),
}<load_from_csv>
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_final = MultiCNN(make_CNN, num_models=15)
 multiCNN_final.fit_generator(datagen, X_train, y_train, verbose = 0, early_stop=True )
def generate_predictions(model_path, max_len):
model = AutoModelForSequenceClassification.from_pretrained(model_path)
 tokenizer = AutoTokenizer.from_pretrained(model_path)
 model.to("cuda")
 model.eval()
df = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 dataset = Dataset(text=df.text.values, tokenizer=tokenizer, max_len=max_len)
 data_loader = torch.utils.data.DataLoader(
 dataset, batch_size=32, num_workers=4, pin_memory=True, shuffle=False 
)
 final_output = []
for b_idx, data in enumerate(data_loader):
with torch.no_grad() :
for key, value in data.items() :
data[key] = value.to("cuda")
 output = model(**data)
 output = torch.nn.functional.softmax(output.logits, dim=1)
 output = output.detach().cpu().numpy() [:, 1]
output =(output >= 0.6 ).astype(int ).tolist()
final_output.extend(output)
 torch.cuda.empty_cache()
return np.array(final_output )<predict_on_test>
multiCNN_final.acuracia(X_test, y_test )
preds = generate_predictions("abhishek/autonlp-fred2-2682064", max_len=64 )<save_to_csv>
submission = pd.read_csv(digits_PATH+'sample_submission.csv', index_col=0)
 test = pd.read_csv(digits_PATH+'test.csv')/255
test = test.values.reshape(-1,28, 28, 1 )
sample = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 sample.target = preds
sample.to_csv("submission.csv", index=False )<count_values>
submission['Label'] = np.argmax(multiCNN_final.predict(test), axis=1 )
sample.target.value_counts()<import_modules>
submission.Label.value_counts()
<load_from_csv><EOS> 
submission.to_csv('submission.csv' )