Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
!pip install transformers==3.5.1
!pip install pyspellchecker
!pip install -U joblib textblob
!python -m textblob.download_corpora
data_train_file = ".. /input/Kannada-MNIST/train.csv"
data_test_file = ".. /input/Kannada-MNIST/test.csv"
df_train = pd.read_csv(data_train_file)
 df_test = pd.read_csv(data_test_file)
 submissions = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv" )<define_variables>
import pandas as pd
import torchtext
from transformers import BertTokenizer, BertForMaskedLM, BertConfig
import transformers
import torch
from torch.utils.data import Dataset, DataLoader
from torch import optim
from torch import cuda
from sklearn.model_selection import train_test_split
import re
import string
from joblib import Parallel, delayed
from textblob import TextBlob
from textblob.translate import NotTranslated
from time import sleep
def get_features_labels(df):
labels = df['label'].values
features = df.values[:, 1:]/255
return features, labels<train_model>
train_val_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
df_train['label'].value_counts()
print(df_test.shape)
 Y_train_1hot = tf.keras.utils.to_categorical(train_labels)
 print(Y_train_1hot.shape)
 print("
")
 X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train_1hot, random_state = 42, test_size = 0.05)
 print(X_train.shape)
 print(Y_train.shape)
 print(X_val.shape)
 print(Y_val.shape )<choose_model_class>
train_val_df = train_val_df.loc[:,["text","target"]]
test_df = test_df.loc[:,["text"]]
test_df["target"] = [0]*len(test_df["text"] )
model_arch = {}
learning_rate_reduction = ReduceLROnPlateau(monitor = 'val_acc',
patience = 3,
verbose = 1,
factor = 0.3,
min_lr = 0.00001)
<choose_model_class>
check_df = train_val_df
<train_model>
languages = ["de"]
parallel = Parallel(n_jobs=-1, backend="threading", verbose=5 )
generated_data = ImageDataGenerator(
 rotation_range = 20,
shear_range = 0.1,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1)
 generated_data.fit(X_train )<choose_model_class>
def translate_text(comment, language):
if hasattr(comment, "decode"):
comment = comment.decode("utf-8")
 text = TextBlob(comment)
 try:
text = text.translate(to=language)
 sleep(2.0)
 text = text.translate(to="en")
 sleep(2.0)
 except NotTranslated:
pass
return str(text )
model_arch['cnn'] = [
tf.keras.layers.Reshape(input_shape =(28, 28, 1), target_shape =(28, 28, 1)) ,
tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),
tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D(( 2, 2), strides = 1),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Conv2D(filters = 64, kernel_size = 3, activation='relu', padding='same'),
tf.keras.layers.Conv2D(filters = 64, kernel_size = 3, activation='relu', padding='same'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D(( 2, 2), strides = 2),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),
tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D(( 2, 2), strides = 2),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation='softmax')
 ] 
<choose_model_class>
comments_list = check_df["text"]
for language in languages:
print('Translate comments using "{0}" language'.format(language))
translated_data = parallel(delayed(translate_text )(comment, language)for comment in comments_list)
 check_df['text'] = translated_data
result_path = os.path.join("train_val_" + language + ".csv")
 check_df.to_csv(result_path, index=False )
model = tf.keras.Sequential(model_arch['cnn'])
 initial_learningrate=2e-3
user_optimizer = RMSprop(lr=initial_learningrate)
 model.compile(loss = 'categorical_crossentropy',
optimizer = user_optimizer,
metrics = ['accuracy'])
 model.summary()<define_variables>
train_val_de_df = pd.read_csv("./train_val_de.csv")
 train_concat_df = train_val_de_df
User_batch_size = 256
EPOCHS = 24<train_model>
print(train_concat_df )
history = model.fit_generator(generated_data.flow(X_train, Y_train, batch_size = User_batch_size),
epochs = EPOCHS, validation_data =(X_val, Y_val),
shuffle = True,
verbose = 1, callbacks = [learning_rate_reduction] )<save_to_csv>
train_val_df = pd.concat([train_val_df,train_concat_df] )
predictions = model.predict(X_test)
 predictions = np.argmax(predictions, axis = 1)
 submissions['label'] = predictions
submissions.to_csv("submission.csv", index = False, header = True)
<set_options>
tokenizer = BertTokenizer.from_pretrained('bert-base-cased' )
plt.ion()<define_variables>
print(torch.__version__ )
dataset_path = '/kaggle/input/Kannada-MNIST/'
output_path = '/kaggle/working/'<set_options>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_URL(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_URL(x))
def remove_html(text):
html = re.compile(r'<.*?>')
 return html.sub(r'',text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_html(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_html(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x: remove_emoji(x))
test_df['text'] = test_df['text'].apply(lambda x: remove_emoji(x))
def remove_punct(text):
table = str.maketrans('','', string.punctuation)
 return text.translate(table)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_punct(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_punct(x))
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : correct_spellings(x))
test_df['text'] = test_df['text'].apply(lambda x : correct_spellings(x))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print('Using %r' % device)
 batch_size = 1024
num_workers = 4
num_folds = 4
num_epochs = 50<load_from_csv>
test_df.to_csv("test.tsv", sep='\t', index=False, header=None)
 print(test_df.shape)
 train_val_df.to_csv("train_eval.tsv", sep='\t', index=False, header=None)
 print(train_val_df.shape)
csv_cache = {}
def read_csv(path):
if path in csv_cache:
return csv_cache[path]
else:
frame = pd.read_csv(path)
 csv_cache[path] = frame
return frame
class MNIST(torch.utils.data.Dataset):
def __init__(self, *paths, train=True, transform=None, split=None):
self.train = train
self.transform = transform
values = pd.concat(read_csv(path)for path in paths ).values
if train:
self.labels = values[:, 0].astype('int64')
 self.images = values[:, 1:].astype('float32' ).reshape(-1, 28, 28)
 else:
self.images = values.astype('float32')[:, 1:].reshape(-1, 28, 28)
 if split is not None:
if train:
self.labels = self.labels[split]
self.images = self.images[split]
def __len__(self):
return self.images.shape[0]
def __getitem__(self, key):
if torch.is_tensor(key):
key = tuple(key.tolist())
 if isinstance(key, tuple):
raise NotImplementedError
elif isinstance(key, slice):
image = self.images[key, :, :].copy()
if self.transform is not None:
for i in range(image.shape[0]):
image[i] = self.transform(image[i])
 elif isinstance(key, int):
image = self.images[key]
if self.transform is not None:
image = self.transform(image)
 if self.train:
label = self.labels[key]
return image, label
else:
return image<save_to_csv>
max_length = 100
def tokenizer_100(input_text):

 return tokenizer.encode(input_text, max_length=100, return_tensors='pt')[0]
TEXT = torchtext.data.Field(sequential=True, tokenize=tokenizer_100, use_vocab=False, lower=False,
include_lengths=True, batch_first=True, fix_length=max_length, pad_token=0)
 LABEL = torchtext.data.Field(sequential=False, use_vocab=False )
augmented_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomAffine(degrees=10, translate=(0.25, 0.25),
scale=(0.9, 1.1), shear=10,
fillcolor=0),
transforms.ToTensor() ,
transforms.Normalize(mean=(128,), std=(128,)) ,
])
 transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=(128,), std=(128,)) ,
])
 trainsets = []
trainloaders = []
for i in range(num_folds):
trainset = MNIST(os.path.join(dataset_path, 'train.csv'), train=True,
transform=augmented_transform,
split=np.arange(i * 60000 // num_folds,(i + 1)* 60000 // num_folds))
trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,
shuffle=True, num_workers=num_workers,
pin_memory=True)
 trainsets.append(trainset)
 trainloaders.append(trainloader)
 testset = MNIST(os.path.join(dataset_path, 'test.csv'), train=False,
transform=transform)
 testloader = torch.utils.data.DataLoader(testset, batch_size=4,
shuffle=False, num_workers=num_workers,
pin_memory=True)
 valset = MNIST(os.path.join(dataset_path, 'Dig-MNIST.csv'), train=True,
transform=transform)
 valloader = torch.utils.data.DataLoader(valset, batch_size=batch_size,
shuffle=False, num_workers=num_workers,
pin_memory=True )<choose_model_class>
dataset_train_eval, dataset_test = torchtext.data.TabularDataset.splits(
 path='.', train='./train_eval.tsv', test='./test.tsv', format='tsv', fields=[('Text', TEXT),('Label', LABEL)] )
class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(64, eps=1e-5, momentum=0.1),
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(64, eps=1e-5, momentum=0.1),
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(64, eps=1e-5, momentum=0.1),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Dropout(0.2),
nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(128, eps=1e-5, momentum=0.1),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(128, eps=1e-5, momentum=0.1),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(128, eps=1e-5, momentum=0.1),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Dropout(0.2),
nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(256, eps=1e-5, momentum=0.1),
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(0.1),
nn.BatchNorm2d(256, eps=1e-5, momentum=0.1),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Dropout(0.2), 
)
 self.dense = nn.Sequential(
 nn.Linear(2304, 256),
nn.LeakyReLU(0.1),
nn.BatchNorm1d(256),
nn.Linear(256, 10), 
)
 def forward(self, x):
x = self.conv1(x)
 x = x.flatten(start_dim=1)
 x = self.dense(x)
 return x<train_model>
print(tokenizer.convert_ids_to_tokens(item.Text.tolist()))
print(int(item.Label))
models = []
for _ in range(num_folds):
model = Model().to(device)
 print(model(iter(trainloader ).next() [0].to(device)).argmax(1 ).tolist())
 models.append(model)
 models<choose_model_class>
batch_size = 32
dl_train = torchtext.data.Iterator(
 dataset_train, batch_size=batch_size, train=True)
 dl_eval = torchtext.data.Iterator(
 dataset_eval, batch_size=batch_size, train=False, sort=False)
 dl_test = torchtext.data.Iterator(
 dataset_test, batch_size=batch_size, train=False, sort=False)
 dataloaders_dict = {"train": dl_train, "val": dl_eval}
criterion = nn.CrossEntropyLoss()
optimizers = [torch.optim.RMSprop(model.parameters() , lr=0.002,
alpha=0.9, momentum=0.1,
eps=1e-7, centered=True)
 for model in models]
schedulers = [torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',
factor=0.25, patience=2,
verbose=True, min_lr=0.00001)
 for optimizer in optimizers]<train_model>
print(transformers.__version__ )
histories = []
for k,(optimizer, model)in enumerate(zip(optimizers, models)) :
schedulers[k]
histories.append([])
 if k > 0:
print()
print()
print()
print(f'Model fold {k + 1}')
 trainsets[k].transform = transform
for epoch in range(num_epochs):
sample = 0
running_total = running_errors = running_loss = 0
epoch_total = epoch_errors = epoch_loss = 0
model.train()
for i, trainloader in enumerate(trainloaders):
if i == k:
continue
for images, labels in trainloader:
optimizer.zero_grad()
outputs = model(images.to(device))
loss = criterion(outputs, labels.to(device))
loss.backward()
optimizer.step()
total = labels.size(0)
 errors =(outputs.argmax(1 ).cpu() != labels ).sum().item()
loss = loss.item()
running_total += total
running_errors += errors
running_loss += loss
epoch_total += total
epoch_errors += errors
epoch_loss += loss
if sample % 10 == 9:
print(f'\rEpoch {epoch + 1}, sample {(sample + 1)* batch_size:6d}, '
f'loss {running_loss / running_total:.5f}, '
f'acc {(1 - running_errors / running_total)* 100:3.2f}',
end='')
 running_total = running_errors = running_loss = 0
sample += 1
model.eval()
total = 0
corrects = 0
for i,(images, labels)in enumerate(trainloaders[k]):
total += labels.size(0)
 corrects +=(model(images.to(device)).argmax(1)== labels.to(device)).sum().cpu().item()
print(f'\r{total} / {len(valset)} inferred', end='')
 print('\r', end='')
 print(f'Epoch {epoch + 1} ')
 print(f'- Val acc: {100 * corrects / total:3.2f}%')
 print(f'- Train acc: {100 *(1 - epoch_errors / epoch_total):3.2f}%, '
f'loss: {epoch_loss:.5f}')
 histories[k].append(( corrects / total, 1 - epoch_errors / epoch_total))
schedulers[k].step(epoch_loss)
 trainsets[k].transform = augmented_transform<save_model>
model = BertModel.from_pretrained('bert-base-cased' )
for i, model in enumerate(models):
torch.save(model.state_dict() , os.path.join(output_path, f'model-{i + 1}.pt'))<prepare_output>
class BertForTwitter(nn.Module):
def __init__(self):
super(BertForTwitter, self ).__init__()
self.bert = model
self.cls = nn.Linear(in_features=768, out_features=9)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids):

 result = self.bert(input_ids)
 vec_0 = result[0]
vec_0 = vec_0[:, 0, :]
vec_0 = vec_0.view(-1, 768)
 output = self.cls(vec_0)
 return output
submission = []
for images in testloader:
predictions = sum(model(images.to(device)) for model in models)
 submission.extend(predictions.argmax(1 ).tolist())
 len(submission )<save_to_csv>
net = BertForTwitter()
net.train()
print('ネットワーク設定完了' )
df = pd.DataFrame.from_records(np.array(submission ).reshape(-1, 1))
df.to_csv(os.path.join(output_path, 'submission.csv'),
index_label='id', header=['label'] )<import_modules>
for param in net.parameters() :
param.requires_grad = False
for param in net.bert.encoder.layer[-1].parameters() :
param.requires_grad = True
for param in net.cls.parameters() :
param.requires_grad = True
import numpy as np
import pandas as pd
import os
import struct
import matplotlib.pyplot as plt
import keras
from keras.layers import *
from keras.models import Sequential, load_model
from keras.optimizers import *
from sklearn.preprocessing import MinMaxScaler
from keras.callbacks import CSVLogger, ModelCheckpoint
from keras.preprocessing.image import ImageDataGenerator<load_from_csv>
optimizer = optim.Adam([
{'params': net.bert.encoder.layer[-1].parameters() , 'lr': 5e-5},
{'params': net.cls.parameters() , 'lr': 1e-4}
])
 criterion = nn.CrossEntropyLoss()
train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')
 valid_part = 10
test_id = test.id
test = test.drop('id', axis=1)
 y_train = train.label
x_train = train.drop('label', axis=1)
 train_size = int(x_train.shape[0] / valid_part *(valid_part - 1))
x_valid, y_valid = x_train[train_size:], y_train[train_size:]
x_train, y_train = x_train[:train_size], y_train[:train_size]<normalization>
def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):
max_acc = 0
Stop_flag = False
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("使用デバイス：", device)
 print('-----start-------')
 net.to(device)
 torch.backends.cudnn.benchmark = True
batch_size = dataloaders_dict["train"].batch_size
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
iteration = 1
for batch in(dataloaders_dict[phase]):
inputs = batch.Text[0].to(device)
 labels = batch.Label.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 loss = criterion(outputs, labels)
 _, preds = torch.max(outputs, 1)
 if phase == 'train':
loss.backward()
optimizer.step()
if(iteration % 50 == 0):
acc =(torch.sum(preds == labels.data)
 ).double() /batch_size
print('イテレーション {} || Loss: {:.4f} || 10iter.|| 本イテレーションの正解率：{}'.format(
 iteration, loss.item() , acc))
iteration += 1
epoch_loss += loss.item() * batch_size
epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_acc = epoch_corrects.double(
)/ len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,
phase, epoch_loss, epoch_acc))
if phase == "val":
if epoch_acc < max_acc:
count += 1
if count >= 3:
Stop_flag = True
else:
count = 0
max_acc = epoch_acc
print(count)
 if Stop_flag:
break
return net
scaler = MinMaxScaler(feature_range=(-1, 1))
rows, cols = 28, 28
x_train = x_train.astype('float32')
 test = test.astype('float32')
 scaler.fit(x_train)
 x_train = scaler.transform(x_train)
 x_valid = scaler.transform(x_valid)
 test = scaler.transform(test)
 print(x_train.min() , x_train.max())
 x_train = x_train.reshape(x_train.shape[0], rows, cols, 1)
 test = test.reshape(test.shape[0], rows, cols, 1)
 x_valid = x_valid.reshape(x_valid.shape[0], rows, cols, 1)
 y_train = keras.utils.to_categorical(y_train, 10)
 y_valid = keras.utils.to_categorical(y_valid, 10 )<choose_model_class>
num_epochs = 100
net_trained = train_model(net, dataloaders_dict,
criterion, optimizer, num_epochs=num_epochs )
train_datagen = ImageDataGenerator(rotation_range = 10,
shear_range = 0.1,
width_shift_range = 0.25,
height_shift_range = 0.25,
zoom_range = 0.25,
horizontal_flip = False)
 epochs = 40
batch_size = 1024
model = Sequential()
model.add(Conv2D(64,
kernel_size=(5, 5),
input_shape=(28, 28, 1),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(Conv2D(64,
kernel_size=(3, 3),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(Conv2D(64,
kernel_size=(3, 3),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(128,
kernel_size=(5, 5),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(Conv2D(128,
kernel_size=(3, 3),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(Conv2D(128,
kernel_size=(3, 3),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(256,
kernel_size=(5, 5),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(Conv2D(256,
kernel_size=(3, 3),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(Conv2D(256,
kernel_size=(3, 3),
padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(gamma_initializer='uniform', epsilon=0.0001, momentum=0.15))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.001),
metrics=['accuracy'])
 model.fit_generator(
 train_datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=100,
epochs=epochs,
callbacks=[CSVLogger('log.csv'), ModelCheckpoint('model.h5', save_best_only=True)],
validation_data=(x_valid, y_valid),
validation_steps=50,
verbose=1)
 model = load_model('model.h5')
 label = model.predict(test)
 label = np.argmax(label, axis=1)
 df = pd.DataFrame({'id': test_id, 'label': label})
 df.to_csv('submission.csv', index=False )<define_variables>
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = ans_list
sample_submission
METRICS = {
'accuracy': {
'f': accuracy_score,
'args': {}
},
}
NORM_MEAN = [0.485, 0.456, 0.406]
NORM_STD = [0.229, 0.224, 0.225]
def make_image_label_grid(images, labels=None, class_names=None):
channels = images.shape[1]
if channels not in(3, 1):
raise ValueError("Images must have 1 or 3 channels")
 mean = NORM_MEAN if channels == 3 else [sum(NORM_MEAN)/ 3]
std = NORM_STD if channels == 3 else [sum(NORM_STD)/ 3]
mean = torch.tensor(mean)
 std = torch.tensor(std)
 mean =(-mean / std ).tolist()
std =(1.0 / std ).tolist()
img_grid = torchvision.utils.make_grid(images)
 img_grid = F.normalize(img_grid, mean=mean, std=std)
 return img_grid
def make_image_label_figure(images, labels=None, class_names=None):
channels = images.shape[1]
if channels not in(3, 1):
raise ValueError("Images must have 1 or 3 channels")
 mean = NORM_MEAN if channels == 3 else [sum(NORM_MEAN)/ 3]
std = NORM_STD if channels == 3 else [sum(NORM_STD)/ 3]
mean = torch.tensor(mean)
 std = torch.tensor(std)
 mean =(-mean / std ).tolist()
std =(1.0 / std ).tolist()
n = int(math.sqrt(len(images)))
 figure = plt.figure(figsize=(n, n))
figure.subplots_adjust(hspace=0.4, wspace=0.4)
 for i in range(n*n):
image, label = images[i],(0 if labels is None else labels[i])
 image = F.normalize(image, mean=mean, std=std)
 image = image.permute(1, 2, 0)
 image = torch.squeeze(image)
 image =(image * 255 ).int()
plt.subplot(n, n, i + 1, title='NA' if class_names is None else class_names[label])
 plt.xticks([])
 plt.yticks([])
 plt.grid(False)
 plt.imshow(image, cmap='gray' if channels == 1 else None)
 return figure
class Transforms(transforms.Compose):
def __init__(self, in_channels=1, out_channels=1, size=(32, 32)) :
if out_channels not in(3, 1)or in_channels not in(3, 1):
raise ValueError("Images must have 1 or 3 channels")
 mean = NORM_MEAN if out_channels == 3 else [sum(NORM_MEAN)/ 3]
std = NORM_STD if out_channels == 3 else [sum(NORM_STD)/ 3]
transforms_list = []
if in_channels != out_channels:
transforms_list.append(transforms.Grayscale(out_channels))
transforms_list.extend([
transforms.Resize(size),
transforms.ToTensor() ,
transforms.Normalize(mean=mean, std=std)])
 super(Transforms, self ).__init__(transforms_list)
 class TrainTransforms(transforms.Compose):
def __init__(self, in_channels=1, out_channels=1, size=(32, 32), random_crop=False, random_affine=False,
horizontal_flip=False, color_jitter=False, random_erasing=False):
if out_channels not in(3, 1)or in_channels not in(3, 1):
raise ValueError("Images must have 1 or 3 channels")
 mean = NORM_MEAN if out_channels == 3 else [sum(NORM_MEAN)/ 3]
std = NORM_STD if out_channels == 3 else [sum(NORM_STD)/ 3]
transforms_list = []
if in_channels != out_channels:
transforms_list.append(transforms.Grayscale(out_channels))
if random_affine:
transforms_list.append(transforms.RandomAffine(degrees=10.0, translate=(0.25, 0.25),
shear=(-10, 10, -10, 10)))
 if random_crop:
transforms_list.append(transforms.RandomResizedCrop(size, scale=(0.9, 1.1), ratio=(0.75, 1.33)))
 else:
transforms_list.append(transforms.Resize(size))
if horizontal_flip:
transforms_list.append(transforms.RandomHorizontalFlip(p=0.5))
if color_jitter:
transforms_list.append(transforms.ColorJitter(brightness=(0.75, 1.5), contrast=(0.75, 1.5),
saturation=(0.75, 1.5), hue=(-0.1, 0.1)))
 transforms_list.append(transforms.ToTensor())
 if random_erasing:
transforms_list.append(transforms.RandomErasing(p=0.2, scale=(0.02, 0.2), ratio=(0.3, 3.3), value=0))
transforms_list.append(transforms.Normalize(mean=mean, std=std))
super(TrainTransforms, self ).__init__(transforms_list)
 class TrainerProgressBar(tqdm):
def __init__(self, desc=None, total=10, unit='it', position=None):
super(TrainerProgressBar, self ).__init__(
 desc=desc, total=total, leave=True, unit=unit, position=position, dynamic_ncols=True 
)
 def reset(self, total=None, desc=None, ordered_dict=None):
self.last_print_n = self.n = 0
self.last_print_t = self.start_t = self._time()
if total is not None:
self.total = total
super(TrainerProgressBar, self ).refresh()
if desc is not None:
super(TrainerProgressBar, self ).set_description(desc)
 if ordered_dict is not None:
super(TrainerProgressBar, self ).set_postfix(ordered_dict)
 def update(self, desc=None, ordered_dict=None, n=1):
if desc is not None:
super(TrainerProgressBar, self ).set_description(desc)
 if ordered_dict is not None:
super(TrainerProgressBar, self ).set_postfix(ordered_dict)
 super(TrainerProgressBar, self ).update(n)
 class PyTorchTrainer(object):
def __init__(self, device=None, metrics=None, epoch_callback=None, batch_callback=None):
self.device = torch.device(device)if device else torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 self.metrics = metrics or METRICS
self.epoch_callback = epoch_callback
self.batch_callback = batch_callback
self.train_pb = None
self.epoch_train_pb = None
self.epoch_val_pb = None
def train(self, model, optimizer, loss_criterion, train_data_loader, val_data_loader, scheduler=None,
epochs=10):
print(f"=============================== Training NN ===============================")
 print(f"== Epochs: {epochs:6d}")
 print(f"== Train batch size: {train_data_loader.batch_size:6d}")
 print(f"== Train batches: {len(train_data_loader):6d}")
 print(f"== Validate batch size: {val_data_loader.batch_size:6d}")
 print(f"== Validate batches: {len(val_data_loader):6d}")
 print(f"===========================================================================")
 self.train_pb = TrainerProgressBar(desc=f'== Epoch {1}', total=epochs, unit='epoch', position=0)
 self.epoch_train_pb = TrainerProgressBar(desc=f'== Train {1}', total=len(train_data_loader),
unit='batch', position=1)
 self.epoch_val_pb = TrainerProgressBar(desc=f'== Val {1}', total=len(val_data_loader), unit='batch',
position=2)
 self.train_pb.reset(total=epochs)
 self.epoch_train_pb.reset(total=len(train_data_loader))
self.epoch_val_pb.reset(total=len(val_data_loader))
for epoch in range(epochs):
train_loss, predictions, targets, inputs = self.forward_batches(model, optimizer, loss_criterion,
train_data_loader, epoch, train=True)
 val_loss, predictions, targets, inputs = self.forward_batches(model, optimizer, loss_criterion,
val_data_loader, epoch, train=False)
 if scheduler:
if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):
scheduler.step(train_loss, epoch=epoch)
 else:
scheduler.step(epoch=epoch)
 metrics_dict = {'lr': optimizer.param_groups[0]['lr'],
'loss': {'train': round(train_loss, 4), 'val': round(val_loss, 4)}}
metrics_dict.update(self.metrics_dict(predictions, targets))
if self.epoch_callback:
self.epoch_callback(epoch, epochs, predictions, targets, inputs, metrics_dict)
 self.train_pb.update(desc=f'== Epoch {epoch+1}', ordered_dict=metrics_dict)
 self.train_pb.close()
self.epoch_train_pb.close()
self.epoch_val_pb.close()
print(f"===========================================================================")
 def forward_batches(self, model, optimizer, loss_criterion, data_loader, epoch, train=True):
if train:
model.train()
else:
model.eval()
avg_loss_value = 0
all_predictions = None
all_targets = None
all_inputs = None
batches = len(data_loader)
 if train:
self.epoch_train_pb.reset(batches, f"== Train {epoch+1}")
 else:
self.epoch_val_pb.reset(batches, f"== Val {epoch+1}")
 for batch_i, data in enumerate(data_loader, 1):
loss_value, predictions, targets, inputs = self.forward_batch(model, optimizer, loss_criterion, data,
train=train)
 avg_loss_value += loss_value
all_predictions = predictions if all_predictions is None else torch.cat(( all_predictions, predictions))
all_targets = targets if all_targets is None else torch.cat(( all_targets, targets))
all_inputs = inputs if all_inputs is None else torch.cat(( all_inputs, inputs))
metrics_dict = {'lr': optimizer.param_groups[0]['lr'], 'loss': avg_loss_value/batch_i}
metrics_dict.update(self.metrics_dict(all_predictions, all_targets))
if self.batch_callback:
self.batch_callback(train, epoch, batch_i, batches, predictions, targets, inputs, metrics_dict)
 if train:
self.epoch_train_pb.update(ordered_dict=metrics_dict)
 else:
self.epoch_val_pb.update(ordered_dict=metrics_dict)
 avg_loss_value /= batches
return avg_loss_value, all_predictions, all_targets, all_inputs
def forward_batch(self, model, optimizer, loss_criterion, batch_data, train=True):
inputs, targets = batch_data
inputs = inputs.to(self.device)
 targets = targets.to(self.device)
 with torch.set_grad_enabled(train):
outputs = model.forward(inputs)
 loss = loss_criterion(outputs, targets)
 if train:
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_value = loss.item()
predictions = outputs.argmax(dim=1 ).data.cpu()
targets = targets.data.cpu()
inputs = inputs.data.cpu()
return loss_value, predictions, targets, inputs
def metrics_dict(self, predictions, targets):
d = {}
for metric_name in self.metrics:
metric_value = self.metrics[metric_name]['f'](predictions, targets, **self.metrics[metric_name]['args'])
 d[metric_name] = metric_value
return d<import_modules>
sample_submission.to_csv("submission_plus.csv", index=False )