Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<load_from_csv>
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
def load_and_prec() :
train = pd.read_csv(TRAIN_DATA, index_col='id')
 test = pd.read_csv(TEST_DATA, index_col='id')
 train['comment_text'] = train['comment_text'].str.lower()
test['comment_text'] = test['comment_text'].str.lower()
train['comment_text'] = train['comment_text'].apply(replace_typical_misspell)
 test['comment_text'] = test['comment_text'].apply(replace_typical_misspell)
 train['comment_text'] = train['comment_text'].apply(clean_text)
 test['comment_text'] = test['comment_text'].apply(clean_text)
 train['comment_text'] = train['comment_text'].apply(clean_numbers)
 test['comment_text'] = test['comment_text'].apply(clean_numbers)
 train_x = train['comment_text'].fillna('_
test_x = test['comment_text'].fillna('_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x))
train_x = tokenizer.texts_to_sequences(train_x)
 test_x = tokenizer.texts_to_sequences(test_x)
 train_x = pad_sequences(train_x, maxlen=maxlen)
 test_x = pad_sequences(test_x, maxlen=maxlen)
 train_y =(train['target'].values > 0.5 ).astype(int)
 np.random.seed(seed)
 train_idx = np.random.permutation(len(train_x))
train_x = train_x[train_idx]
train_y = train_y[train_idx]
return train_x, train_y, test_x, tokenizer.word_index<choose_model_class>
old_result_df = pd.read_csv('.. /input/kmnist-data/Old_training_result.csv')
 old_result_df.head(5 )
class Net(nn.Module):
def __init__(self, embedding_matrix):
super(Net, self ).__init__()
lstm_hidden_size = 120
gru_hidden_size = 60
self.gru_hidden_size = gru_hidden_size
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, lstm_hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(lstm_hidden_size*2, gru_hidden_size, bidirectional=True, batch_first=True)
 self.linear = nn.Linear(gru_hidden_size*6, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.unsqueeze(h_embedding.transpose(1, 2), 2)
 h_embedding = torch.squeeze(self.embedding_dropout(h_embedding)).transpose(1, 2)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, hh_gru = self.gru(h_lstm)
 hh_gru = hh_gru.view(-1, self.gru_hidden_size*2)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( hh_gru, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<set_options>
display_single_fold_val_accuracy(old_result_df, 1 )
warnings.filterwarnings('ignore' )<load_pretrained>
display_single_fold_val_accuracy(old_result_df, 2 )
with timer('load data'):
train_x, train_y, test_x, word_index = load_and_prec()
embedding_matrix = load_fasttext(word_index )<import_modules>
display_single_fold_val_accuracy(old_result_df, 3 )
import random
import os<categorify>
new_result_df = pd.read_csv('.. /input/kmnist-data/New_training_result.csv')
 new_result_df.head(5 )
_camel_re1 = re.compile(' (.)([A-Z][a-z]+)')
 _camel_re2 = re.compile('([a-z0-9] )([A-Z])')
 def camel2snake(name):
s1 = re.sub(_camel_re1, r'\1_\2', name)
 return re.sub(_camel_re2, r'\1_\2', s1 ).lower()<train_model>
warnings.filterwarnings('ignore')
 main_dir = '.. /input/Kannada-MNIST/'
tf.keras.__version__
class TrainEvalCallback(Callback):
def begin_fit(self):
self.run.n_epochs = 0
self.run.n_iter = 0
def after_batch(self):
if not self.in_train: return
self.run.n_epochs+=1./self.iters
self.run.n_iter+=1
def begin_epoch(self):
self.run.n_epochs = self.epoch
self.model.train()
self.run.in_train = True
def begin_validate(self):
self.model.eval()
self.run.in_train = False
class CancelTrainException(Exception): pass
class CancelEpochException(Exception): pass
class CancelBatchException(Exception): pass<import_modules>
pretrain_weights_path = [
'.. /input/kmnist-data/0_model.hdf5',
'.. /input/kmnist-data/1_model.hdf5',
'.. /input/kmnist-data/2_model.hdf5',
'.. /input/kmnist-data/3_model.hdf5',
'.. /input/kmnist-data/4_model.hdf5',
'.. /input/kmnist-data/5_model.hdf5',
'.. /input/kmnist-data/6_model.hdf5',
'.. /input/kmnist-data/7_model.hdf5',
]
uptrain = False
submit = True
num_classes = 10
num_features =(28,28,1)
 batch_size = 1024
lr = 3e-4
epochs = 30
k_fold_split = 8
from collections import Iterable<categorify>
train_df = pd.read_csv(main_dir + 'train.csv')
 valid_df = pd.read_csv(main_dir + 'Dig-MNIST.csv')
 test_df = pd.read_csv(main_dir + 'test.csv')
 train_df.head(5 )
def listify(o):
if o is None: return []
if isinstance(o, list): return o
if isinstance(o, str): return [o]
if isinstance(o, Iterable): return list(o)
 return [o]<find_best_params>
train_x = train_x.reshape(-1,28,28,1)
 train_y = tf.keras.utils.to_categorical(train_y, num_classes)
 test_x = test_x.reshape(-1,28,28,1 )
class Runner() :
def __init__(self, cbs=None, cb_funcs=None):
cbs = listify(cbs)
 for cbf in listify(cb_funcs):
cb = cbf()
setattr(self, cb.name, cb)
 cbs.append(cb)
 self.stop, self.cbs = False, [TrainEvalCallback() ] + cbs
@property
def opt(self): return self.learn.opt
@property
def model(self): return self.learn.model
@property
def loss_func(self): return self.learn.loss_func
@property
def data(self): return self.learn.data
def one_batch(self, xb, yb):
try:
self.xb, self.yb = xb, yb
self('begin_batch')
 self.pred = self.model(self.xb)
 self('after_pred')
 self.loss = self.loss_func(self.pred, self.yb)
 self('after_loss')
 if not self.in_train: return
self.loss.backward()
self('after_backward')
 self.opt.step()
self('after_step')
 self.opt.zero_grad()
except CancelBatchException: self('after_cancel_batch')
 finally: self('after_batch')
 def all_batches(self, dl):
self.iters = len(dl)
 try:
for xb, yb in progress_bar(dl, leave=False): self.one_batch(xb, yb)
 except CancelEpochException: self('after_cancel_epoch')
 def fit(self, epochs, learn):
self.epochs, self.learn, self.loss = epochs, learn, torch.tensor(0.)
try:
for cb in self.cbs: cb.set_runner(self)
 self('begin_fit')
 for epoch in range(epochs):
self.epoch = epoch
if not self('begin_epoch'): self.all_batches(self.data.train_dl)
 with torch.no_grad() :
if not self('begin_validate'): self.all_batches(self.data.valid_dl)
 self('after_epoch')
 except CancelTrainException: self('after_cancel_train')
 finally:
self('after_fit')
 self.learn = None
def __call__(self, cb_name):
res = False
for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name)and res
return res<choose_model_class>
class InputGenerator(tf.keras.utils.Sequence):
def __init__(self,
x,
y=None,
aug=None,
batch_size=128,
training=True):
self.x = x
self.y = y
self.aug = aug
self.batch_size = batch_size
self.training = training
self.indices = range(len(x))
def __len__(self):
return len(self.x)// self.batch_size
def __getitem__(self,index):
batch_indices = self.indices[index * self.batch_size :(index+1)*self.batch_size]
batch_data = self.__get_batch_x(batch_indices)
 if self.training:
batch_label = self.__get_batch_y(batch_indices)
 return batch_data, batch_label
else:
return batch_data
def on_epoch_start(self):
if self.training:
np.random.shuffle(self.indices)
 def __get_batch_x(self, batch_indices):
batch_data = []
for idx in batch_indices:
cur_data = self.x[idx].astype('uint8')
 cur_data = self.aug(image = cur_data)['image']
batch_data.append(cur_data)
 return np.stack(batch_data)/255.0
def __get_batch_y(self, batch_indices):
batch_label = []
for idx in batch_indices:
batch_label.append(self.y[idx])
 return np.stack(batch_label)
 train_aug = albu.Compose([
albu.ShiftScaleRotate(scale_limit=0.2, rotate_limit=15.0, shift_limit=0.15, p=0.5, border_mode=0, value = 0)] 
)
 valid_aug = albu.Compose([] )
class Learner() :
def __init__(self, model, opt, loss_func, data):
self.model, self.opt, self.loss_func, self.data = model, opt, loss_func, data<choose_model_class>
def Precision(y_true, y_pred, epsilon=1e-7):
y_true_f = tf.keras.backend.flatten(y_true)
 y_pred_f = tf.keras.backend.flatten(y_pred)
 y_pred_f = tf.keras.backend.round(y_pred_f)
 TP = tf.keras.backend.sum(y_true_f * y_pred_f)
 FP = tf.keras.backend.sum(( 1-y_true_f)* y_pred_f)
 return TP/(TP+FP+epsilon)
 def Recall(y_true, y_pred, epsilon=1e-7):
y_true_f = tf.keras.backend.flatten(y_true)
 y_pred_f = tf.keras.backend.flatten(y_pred)
 y_pred_f = tf.keras.backend.round(y_pred_f)
 TP = tf.keras.backend.sum(y_true_f * y_pred_f)
 TN = tf.keras.backend.sum(y_true_f *(1-y_pred_f))
return TP/(TP+TN+epsilon )
def get_model(data, lr=0.001):
model = Net(embedding_matrix ).to(device)
 return model, torch.optim.Adam(model.parameters() , lr )<train_model>
def symmetric_cross_entropy(alpha=1.0, beta=1.0, epsilon=1e-7):
def loss(y_true, y_pred):
y_pred_ce = tf.clip_by_value(y_pred, epsilon, 1.0)
 y_true_rce = tf.clip_by_value(y_true, epsilon, 1.0)
 ce = alpha*tf.reduce_mean(-tf.reduce_sum(y_true * tf.math.log(y_pred_ce), axis = -1))
rce = beta*tf.reduce_mean(-tf.reduce_sum(y_pred * tf.math.log(y_true_rce), axis = -1))
return ce + rce
return loss
class DataBunch() :
def __init__(self, train_dl, valid_dl):
self.train_dl, self.valid_dl = train_dl, valid_dl
@property
def train_ds(self): return self.train_dl.dataset
@property
def valid_ds(self): return self.valid_dl.dataset<split>
permutation = np.random.RandomState(2019 ).permutation(len(train_x))
train_x = train_x[permutation]
train_y = train_y[permutation]
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed ).split(train_x, train_y))<set_options>
kf = KFold(n_splits=k_fold_split, random_state=2019)
 model_members = []
check_points_path = []
for idx in range(k_fold_split):
check_points_path.append(str(idx)+ '_model.hdf5')
 for model_index,(train_indices, valid_indices)in enumerate(kf.split(train_x)) :
train_aug_gen = InputGenerator(train_x[train_indices], train_y[train_indices], train_aug, batch_size)
 valid_aug_gen = InputGenerator(train_x[valid_indices], train_y[valid_indices], valid_aug, batch_size)
 model = build_model()
optimizer = tf.keras.optimizers.RMSprop(lr=lr)
 model.compile(optimizer=optimizer, metrics=['accuracy',Precision,Recall], loss=symmetric_cross_entropy())
 steps_per_epoch = len(train_x[train_indices])// batch_size
validation_steps = len(train_x[valid_indices])// batch_size
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5, min_lr=1e-8, mode='min', verbose=2)
 check_point = tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', filepath = check_points_path[model_index], mode='min', save_best_only=True, verbose=2)
 early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)
 training_callbacks = [ check_point, reduce_lr ]
if pretrain_weights_path == None or uptrain == True:
if pretrain_weights_path != None:
print('*'*10,'Load ',model_index,'-fold pretrain weights','*'*10)
 model.load_weights(pretrain_weights_path[model_index])
 print('*'*30,'Training model ', model_index+1,'*'*30)
 print('Train on ', len(train_indices),' data')
 print('Valid on ', len(valid_indices),' data')
 history = model.fit_generator(generator=train_aug_gen,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=valid_aug_gen,
validation_steps=validation_steps,
workers=-1,
verbose=2,
callbacks=training_callbacks)
 print('*'*30,'Validating model ', model_index+1,'*'*30)
 val_gen = InputGenerator(train_x[valid_indices], None, albu.Compose([]), 1, training=False)
 print('
')
 preds = model.predict_generator(val_gen, workers=-1, verbose=1)
 truths = train_y[valid_indices]
preds = np.round(np.array(preds))
truths = np.array(truths)
 valid_results = []
for c in range(num_classes):
valid_results.append(( c+1, np.sum(preds[:,c] * truths[:,c])/ np.sum(truths[:,c])))
 valid_results_df = pd.DataFrame(valid_results, columns=['Class', 'Valid_accuracy'])
 valid_results_df.to_csv('new_ValidationResults'+str(model_index+1)+'.csv', index=False)
 print(valid_results_df)
 del history, val_gen, train_aug_gen, valid_aug_gen, preds, truths, valid_results_df
gc.collect()
else:
print('Load the pretrain weights '+str(model_index+1))
model.load_weights(pretrain_weights_path[model_index])
 model_members.append(model)
 del model
print('
' )
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<create_dataframe>
tta_aug = albu.Compose([
albu.ShiftScaleRotate(scale_limit=0.1, rotate_limit=10, shift_limit=0.10, p=1.0, border_mode=0, value=0)
 ] )
x_test_cuda = torch.tensor(test_x, dtype=torch.long ).to(device)
 test = torch.utils.data.TensorDataset(x_test_cuda)
 test_dl = DataLoader(test, batch_size=batch_size, shuffle=False )<data_type_conversions>
if submit == True:
predictions = np.zeros(( len(test_x), num_classes))
for model_index, model in enumerate(model_members):
print(str(model_index+1)+ '_model predicting')
 normal_generator = InputGenerator(test_x,
aug = albu.Compose([]),
training=False,
batch_size=250)
 aug_generator = InputGenerator(x=test_x,
aug = tta_aug,
training=False,
batch_size=250)
 tta_model = tta_wrapper(model=model,
normal_generator=normal_generator,
aug_generator = aug_generator,
repeats=1)
 predictions += tta_model.predict()
predictions = predictions / k_fold_split
predictions = np.argmax(predictions, axis=1)
 submission = pd.DataFrame({'id' : test_indices, 'label':predictions})
 submission.to_csv('submission.csv', index = False)
 submission.head(5 )
def get_data(train_idx, valid_idx):
x_train_ds = torch.tensor(train_x[train_idx], dtype=torch.long ).to(device)
 y_train_ds = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).to(device)
 x_val_ds = torch.tensor(train_x[valid_idx], dtype=torch.long ).to(device)
 y_val_ds = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).to(device)
 train_ds = torch.utils.data.TensorDataset(x_train_ds, y_train_ds)
 valid_ds = torch.utils.data.TensorDataset(x_val_ds, y_val_ds)
 train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
 valid_dl = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)
 data = DataBunch(train_dl, valid_dl)
 return data<compute_train_metric>
valid_labels = valid_df['label']
valid_datas = valid_df.drop(['label'],axis=1 ).values
valid_datas = valid_datas.reshape(-1,28,28,1)
 valid_predictions = np.zeros(( len(valid_datas), num_classes))
for model_index, model in enumerate(model_members):
print(str(model_index+1)+ '_model predicting')
 normal_generator = InputGenerator(x=valid_datas,
aug = albu.Compose([]),
training=False,
batch_size=160)
 aug_generator = InputGenerator(x=valid_datas,
aug = tta_aug,
training=False,
batch_size=160)
 tta_model = tta_wrapper(model=model,
normal_generator=normal_generator,
aug_generator = aug_generator,
repeats=1)
 valid_predictions += tta_model.predict()
valid_predictions = valid_predictions/k_fold_split
valid_predictions = np.argmax(valid_predictions,axis=1 )
class FocalLoss(nn.Module):
def __init__(self, alpha=1, gamma=4, logits=False, reduce=True):
super(FocalLoss, self ).__init__()
self.alpha = alpha
self.gamma = gamma
self.logits = logits
self.reduce = reduce
def forward(self, inputs, targets):
if self.logits:
BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)
 else:
BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)
 pt = torch.exp(-BCE_loss)
 F_loss = self.alpha *(1-pt)**self.gamma * BCE_loss
if self.reduce:
return torch.mean(F_loss)
 else:
return F_loss
loss_fn = FocalLoss(logits=True )<choose_model_class>
print('Real world validation accuracy : {}'.format(accuracy_score(valid_labels, valid_predictions)) )
def get_learner(train_idx, valid_idx):
data = get_data(train_idx, valid_idx)
 learn = Learner(*get_model(data), loss_fn, data=data)
 return learn<compute_train_metric>
print(classification_report(valid_labels, valid_predictions))